ParallelDofMapping.h 13.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
28
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
29
#include "parallel/StdMpi.h"
30
31
32
33
34

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
35

36
37
  using namespace std;

38
  /** \brief
39
40
41
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
42
43
44
45
46
47
48
49
50
51
52
53
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

54
55
56
57
58
59
60
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
61
62
63
64
65
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

66
    /// Maps a global DOF index to the global matrix index for a specific 
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


89
90
91
92
93
94
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
95
96
  {
  public:
97
98
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
99
    FeSpaceDofMap() 
100
101
102
    {
      ERROR_EXIT("Should not be called!\n");
    }
103
104
     
    /// This is the only valid constructur to be used. 
105
    FeSpaceDofMap(MPI::Intracomm m)
106
      : mpiComm(m),
Thomas Witkowski's avatar
Thomas Witkowski committed
107
108
	sendDofs(NULL),
	recvDofs(NULL),
109
	feSpace(NULL),
110
111
	nLevel(1),
	dofMap(1),
Thomas Witkowski's avatar
Thomas Witkowski committed
112
	needGlobalMapping(false),
113
	hasNonLocalDofs(false),
114
115
116
117
118
119
120
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
      clear();
    }
121
122

    /// Clears all data of the mapping.
123
    void clear();
124
    
125
126
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
127
    MultiIndex& operator[](DegreeOfFreedom d)
128
    {
129
      TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
130

131
      return dofMap[0][d];
132
133
    }
    
134
135
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
136
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
137
    {
138
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
139
      
140
      TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
141
      
142
143
      dofMap[0][dof0].local = dof1;
      nLocalDofs[0]++;
Thomas Witkowski's avatar
Thomas Witkowski committed
144
      if (dof1 != -1)
145
	nRankDofs[0]++;
146
147
    }
    
148
149
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
150
    void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
151
    {
152
      FUNCNAME("FeSpaceDofMap::insert()");
153
      
154
      TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
155
      
156
157
158
      dofMap[0][dof0].local = dof1;
      nLocalDofs[0]++;
      nonRankDofs[0].insert(dof0);
159
160
    }
    
161
    /// Checks if a given DOF is in the DOF mapping.
162
    bool isSet(DegreeOfFreedom dof, int level = 0)
163
    {
164
      return static_cast<bool>(dofMap[level].count(dof));
165
    }
166

167
168
169
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
170
171
    bool isRankDof(DegreeOfFreedom dof)
    {
172
      return !(static_cast<bool>(nonRankDofs[0].count(dof)));
173
    }
174
    
175
    /// Returns number of DOFs in the mapping.
176
177
    unsigned int size()
    {
178
      return dofMap[0].size();
179
180
    }
    
181
    /// Returns the raw data of the mapping.
182
    DofMap& getMap(int level)
183
    {
184
      return dofMap[level];
185
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
186

187
188
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
189

190
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
191
192
193
194
195
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

196
197
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
198
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
199
    {
200
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
201
202
    }

203
    /// Informs the mapping whether a global index must be computed.
204
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
205
    {
206
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
207
208
    }

209
    /// Sets the DOF communicators.
210
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
211
    {
212
213
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
214
215
    }

216
  private:
217
    /// Computes a global mapping from the local one.
218
    void computeGlobalMapping(int level);
219
220
221

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
222
    void computeNonLocalIndices(int level);
223
224
225

  private:
    /// MPI communicator object;
226
    MPI::Intracomm mpiComm;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
227

228
229
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
230

231
232
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
233
234
    const FiniteElemSpace *feSpace;

235
236
237
    /// Number of mesh levels.
    int nLevel;

238
    /// Mapping data from DOF indices to local and global indices.
239
    vector<DofMap> dofMap;
240

241
    /// Set of all DOFs that are in mapping but are not owned by the rank.
242
    vector<std::set<DegreeOfFreedom> > nonRankDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
243

244
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
245
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
246

247
248
249
250
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
251

252
253
  public:
    /// 
254
    vector<int> nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
255
256
  };
  
257
258
259
260
261
262
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
263
  class ParallelDofMapping
264
265
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
266
    ParallelDofMapping() 
267
      : sendDofs(NULL),
268
	recvDofs(NULL),
269
	hasNonLocalDofs(false),
270
271
	needMatIndex(false),
	needMatIndexFromGlobal(false),
272
	nLevel(1),
273
	feSpaces(0),
274
275
276
277
278
279
280
281
282
283
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
      nRankDofs[0] = -1;
      nLocalDofs[0] = -1;
      nOverallDofs[0] = -1;
      rStartDofs[0] = -1;
    } 
284

285
286
287
288
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
289
290
291
292
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
293
294
295
296
297
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
298
    void init(MPI::Intracomm m,
299
	      vector<const FiniteElemSpace*> &fe,
300
	      vector<const FiniteElemSpace*> &uniqueFe,
301
302
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
303
304
305
    
    /// Clear all data.
    void clear();
306
307
308
309
310
311
312

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
313
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
314
      FUNCNAME("ParallelDofMapping::operator[]()");
315

316
317
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
318
319
320
321

      return data.find(feSpace)->second;
    }

322
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
323
    inline int getRankDofs(int level)
324
    {
325
      TEST_EXIT_DBG(nRankDofs[level] >= 0)("Should not happen!\n");
326

327
      return nRankDofs[level];
328
329
    }

330
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
331
    inline int getLocalDofs(int level)
Thomas Witkowski's avatar
Thomas Witkowski committed
332
    {
333
      TEST_EXIT_DBG(nLocalDofs[level] >= 0)("Should not happen!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
334

335
      return nLocalDofs[level];
Thomas Witkowski's avatar
Thomas Witkowski committed
336
337
    }

338
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
339
    inline int getOverallDofs(int level)
340
    {
341
      TEST_EXIT_DBG(nOverallDofs[level] >= 0)("Should not happen!\n");
342

343
      return nOverallDofs[level];
344
345
    }

346
347
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
348
    inline int getStartDofs(int level)
349
    {
350
      TEST_EXIT_DBG(rStartDofs[level] >= 0)("Should not happen!\n");
351

352
      return rStartDofs[level];
353
354
    }

355
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
356
    void update();
357

358
359
360
    /// Update the mapping.
    void update(vector<const FiniteElemSpace*>& feSpaces);

361
362
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
Thomas Witkowski's avatar
Thomas Witkowski committed
363
364
365
366
367
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
    {
      return dofToMatIndex.get(ithComponent, d);
    }

368
369
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
370
371
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
372
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
373
374
375
376

      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
	("Should not happen!\n");

377
      return dofToMatIndex.get(ithComponent, d) - rStartDofs[0];
378
379
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
380
381
382
383
384
385
386
387
388
389
390
391
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

392
393
394
395
396
397
398
    void setComputeMatIndex(bool b, bool global = false)
    {
      needMatIndex = b;
      needMatIndexFromGlobal = global;
    }

    /// Compute local and global matrix indices.
399
    void computeMatIndex(bool globalIndex, int level);
400

401
402
403
404
405
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
406
    int computeRankDofs(int level);
407
408

    /// Compute \ref nLocalDofs.
409
    int computeLocalDofs(int level);
410
411

    /// Compute \ref nOverallDofs.
412
    int computeOverallDofs(int level);
413
414

    /// Compute \ref rStartDofs.
415
    int computeStartDofs(int level);
416

417
  private:
418
    /// MPI communicator object;
419
    MPI::Intracomm mpiComm;
420

421
422
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
423

424
425
426
427
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
428

429
430
431
432
433
434
435
436
437
438
    /// If true, matrix indeces for the stored DOFs are computed, see
    /// \ref computeMatIndex.
    bool needMatIndex;

    /// If matrix indices should be computed, this variable defines if the
    /// mapping from DOF indices to matrix row indices is defined on local
    /// or global DOF indices. If true, the mapping is to specify and to use
    /// on global ones, otherwise on local DOF indices.
    bool needMatIndexFromGlobal;

439
440
    int nLevel;

441
442
443
444
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
445
446
    vector<const FiniteElemSpace*> feSpaces;

447
448
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
449
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
450

451
    /// Number of DOFs owned by rank.
452
    vector<int> nRankDofs;
453
454

    /// Number of DOFs in rank's subdomain.
455
    vector<int> nLocalDofs;
456
457

    /// Number of global DOFs (this value is thus the same on all ranks).
458
    vector<int> nOverallDofs;
459
460

    /// Smallest global index of a DOF owned by the rank.
461
    vector<int> rStartDofs;
462
463
464
465

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
    DofToMatIndex dofToMatIndex;
466
467
468
469
470
  };

}

#endif