ParallelDofMapping.h 14.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
#include "parallel/MeshLevelData.h"
28
29
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
30
#include "parallel/StdMpi.h"
31

32
33
#include <petscis.h>

34
35
36
37
#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
38

39
40
  using namespace std;

41
  /** \brief
42
43
44
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
45
46
47
48
49
50
51
52
53
54
55
56
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

57
58
59
60
61
62
63
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
64
65
66
67
68
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

69
    /// Maps a global DOF index to the global matrix index for a specific 
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

85
86
87
88
89
    /// Returns for a given matrix index the component and (local or global) DOF
    /// index. As the data structure is not made for this kind of reverse
    /// search, this is very slow and should be only used for debugging.
    void getReverse(int rowIndex, int &component, int &dofIndex);

90
91
92
93
94
95
96
  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


97
98
99
100
101
102
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
103
104
  {
  public:
105
106
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
107
    FeSpaceDofMap() 
108
109
110
    {
      ERROR_EXIT("Should not be called!\n");
    }
111
112
     
    /// This is the only valid constructur to be used. 
113
114
    FeSpaceDofMap(MeshLevelData* ld)
      : levelData(ld),
115
	dofComm(NULL),
116
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
117
	needGlobalMapping(false),
118
	hasNonLocalDofs(false)
119
120
121
    {
      clear();
    }
122
123

    /// Clears all data of the mapping.
124
    void clear();
125

126
127
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
128
    MultiIndex& operator[](DegreeOfFreedom d)
129
    {
130
      TEST_EXIT_DBG(dofMap.count(d))("DOF %d is not in map!\n", d);
131

132
      return dofMap[d];
133
134
    }
    
135
136
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
137
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
138
    {
139
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
140
      
141
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
142
      
143
144
      dofMap[dof0].local = dof1;
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
145
      if (dof1 != -1)
146
	nRankDofs++;
147
148
    }
    
149
150
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
151
    void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
152
    {
153
      FUNCNAME("FeSpaceDofMap::insertNonRankDof()");
154
      
155
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
156
      
157
158
159
      dofMap[dof0].local = dof1;
      nLocalDofs++;
      nonRankDofs.insert(dof0);
160
161
    }
    
162
    /// Checks if a given DOF is in the DOF mapping.
163
    bool isSet(DegreeOfFreedom dof)
164
    {
165
      return static_cast<bool>(dofMap.count(dof));
166
    }
167

168
169
170
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
171
    bool isRankDof(DegreeOfFreedom dof)
172
    {
173
      return !(static_cast<bool>(nonRankDofs.count(dof)));
174
    }
175
    
176
    /// Returns number of DOFs in the mapping.
177
    unsigned int size()
178
    {
179
      return dofMap.size();
180
181
    }
    
182
    /// Returns the raw data of the mapping.
183
    DofMap& getMap()
184
    {
185
      return dofMap;
186
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
187

188
189
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
190

191
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
192
193
194
195
196
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

197
198
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
199
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
200
    {
201
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
202
203
    }

204
    /// Informs the mapping whether a global index must be computed.
205
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
206
    {
207
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
208
209
    }

210
    /// Sets the DOF communicators.
211
    void setDofComm(DofComm &dc)
Thomas Witkowski's avatar
Thomas Witkowski committed
212
    {
213
      dofComm = &dc;
Thomas Witkowski's avatar
Thomas Witkowski committed
214
215
    }

216
217
218
219
220
221
    void setMpiComm(MPI::Intracomm &m, int l)
    {
      mpiComm = m;
      meshLevel = l;
    }

222
  private:
223
    /// Computes a global mapping from the local one.
224
    void computeGlobalMapping();
225
226
227

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
228
    void computeNonLocalIndices();
229
230

  private:
231
    MeshLevelData *levelData;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
232

233
234
    /// DOF communicator for all DOFs on interior boundaries.
    DofComm *dofComm;
235

236
237
238
239
    MPI::Intracomm mpiComm;

    int meshLevel;

240
    /// The FE space this mapping belongs to. This is used only the get the
241
    /// correct DOF communicator in \ref dofComm.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
242
243
    const FiniteElemSpace *feSpace;

244
    /// Mapping data from DOF indices to local and global indices.
245
    DofMap dofMap;
246

247
    /// Set of all DOFs that are in mapping but are not owned by the rank.
248
    std::set<DegreeOfFreedom> nonRankDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
249

250
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
251
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
252

253
254
255
256
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
257

258
259
  public:
    /// 
260
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
261
262
  };
  
263
264
265
266
267
268
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
269
  class ParallelDofMapping
270
271
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
272
    ParallelDofMapping() 
273
      : levelData(NULL),
274
	hasNonLocalDofs(false),
275
276
	needMatIndex(false),
	needMatIndexFromGlobal(false),
277
	feSpaces(0),
278
279
280
281
282
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
283
284
285
286
      nRankDofs = -1;
      nLocalDofs = -1;
      nOverallDofs = -1;
      rStartDofs = -1;
287
    } 
288

289
290
291
292
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
293
294
295
296
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
297
298
299
300
301
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
302
    void init(MeshLevelData& mld,
303
	      vector<const FiniteElemSpace*> &fe,
304
	      vector<const FiniteElemSpace*> &uniqueFe,
305
306
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
307
308

    void setMpiComm(MPI::Intracomm &m, int l);
309
310
311
    
    /// Clear all data.
    void clear();
312
313
314

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
315
    void setDofComm(DofComm &dofComm);
316

317
318
319
320
321
322
    void setComputeMatIndex(bool b, bool global = false)
    {
      needMatIndex = b;
      needMatIndexFromGlobal = global;
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
323
324
325
326
327
    inline bool isMatIndexFromGlobal()
    {
      return needMatIndexFromGlobal;
    }

328
329
    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
330
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
331
      FUNCNAME("ParallelDofMapping::operator[]()");
332

333
334
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
335
336
337
338

      return data.find(feSpace)->second;
    }

339
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
340
    inline int getRankDofs()
341
    {
342
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");
343

344
      return nRankDofs;
345
346
    }

347
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
348
    inline int getLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
349
    {
350
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
351

352
      return nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
353
354
    }

355
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
356
    inline int getOverallDofs()
357
    {
358
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");
359

360
      return nOverallDofs;
361
362
    }

363
364
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
365
    inline int getStartDofs()
366
    {
367
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");
368

369
      return rStartDofs;
370
371
    }

372
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
373
    void update();
374

375
376
377
    /// Update the mapping.
    void update(vector<const FiniteElemSpace*>& feSpaces);

378
379
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
380
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
Thomas Witkowski's avatar
Thomas Witkowski committed
381
    {
382
      return dofToMatIndex.get(ithComponent, d);
Thomas Witkowski's avatar
Thomas Witkowski committed
383
384
    }

385
386
387
388
389
390
391
    /// Returns the component number and local/global DOF index for a given
    /// matrix row index. Should be used for debugging only!
    inline void getReverseMatIndex(int index, int &component, int &dofIndex)
    {
      dofToMatIndex.getReverse(index, component, dofIndex);
    }

392
393
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
394
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
395
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
396
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
397

398
      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
399
400
	("Should not happen!\n");

401
      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
402
403
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
404
405
406
407
408
409
410
411
412
413
414
415
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

416
    /// Compute local and global matrix indices.
417
    void computeMatIndex(bool globalIndex);
418

419
420
421
422
    void createIndexSet(IS &is, 
			int firstComponent, 
			int nComponents);

423
424
425
426
427
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
428
    int computeRankDofs();
429
430

    /// Compute \ref nLocalDofs.
431
    int computeLocalDofs();
432
433

    /// Compute \ref nOverallDofs.
434
    int computeOverallDofs();
435
436

    /// Compute \ref rStartDofs.
437
    int computeStartDofs();
438

439
  private:
440
441
442
443
    MPI::Intracomm mpiComm;

    int meshLevel;

444
    MeshLevelData *levelData;
445

446
447
    /// DOF communicator for all DOFs on interior boundaries.
    DofComm *dofComm;
448

449
450
451
452
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
453

454
455
456
457
458
459
460
461
    /// If true, matrix indeces for the stored DOFs are computed, see
    /// \ref computeMatIndex.
    bool needMatIndex;

    /// If matrix indices should be computed, this variable defines if the
    /// mapping from DOF indices to matrix row indices is defined on local
    /// or global DOF indices. If true, the mapping is to specify and to use
    /// on global ones, otherwise on local DOF indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
462
463
464
465
466
    /// In most scenarios the mapping stored on local DOF indices is what we
    /// want to have. Only when periodic boundary conditions are used together
    /// with some global matrix approache, the matrix indices must be stored
    /// also for DOFs that are not part of the local subdomain. Thus, the
    /// mapping will be stored on global DOF indices.
467
468
    bool needMatIndexFromGlobal;

469
470
471
472
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
473
474
    vector<const FiniteElemSpace*> feSpaces;

475
476
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
477
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
478

479
    /// Number of DOFs owned by rank.
480
    int nRankDofs;
481
482

    /// Number of DOFs in rank's subdomain.
483
    int nLocalDofs;
484
485

    /// Number of global DOFs (this value is thus the same on all ranks).
486
    int nOverallDofs;
487
488

    /// Smallest global index of a DOF owned by the rank.
489
    int rStartDofs;
490
491
492

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
493
    DofToMatIndex dofToMatIndex;
494
495
496
497
498
  };

}

#endif