ParallelDofMapping.h 12.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25
26
#include <set>
#include "parallel/DofComm.h"
27
28
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
29
#include "parallel/StdMpi.h"
30
31
32
33
34

#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
35

36
37
  using namespace std;

38
  /** \brief
39
40
41
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
42
43
44
45
46
47
48
49
50
51
52
53
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

54
55
56
57
58
59
60
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
61
62
63
64
65
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

66
    /// Maps a global DOF index to the global matrix index for a specific 
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


89
90
91
92
93
94
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
95
96
  {
  public:
97
98
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
99
    FeSpaceDofMap() 
100
101
102
    {
      ERROR_EXIT("Should not be called!\n");
    }
103
104
     
    /// This is the only valid constructur to be used. 
105
    FeSpaceDofMap(MPI::Intracomm m)
106
      : mpiComm(m),
Thomas Witkowski's avatar
Thomas Witkowski committed
107
108
	sendDofs(NULL),
	recvDofs(NULL),
109
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
110
	needGlobalMapping(false),
111
	hasNonLocalDofs(false),
112
	nRankDofs(0),
Thomas Witkowski's avatar
Thomas Witkowski committed
113
	nLocalDofs(0),
114
	nOverallDofs(0),
115
	rStartDofs(0)
116
    {}
117
118

    /// Clears all data of the mapping.
119
    void clear();
120
    
121
122
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
123
    MultiIndex& operator[](DegreeOfFreedom d)
124
125
126
127
128
129
    {
      TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");

      return dofMap[d];
    }
    
130
131
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
132
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
133
    {
134
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
135
136
137
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
138
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
139
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
140
141
      if (dof1 != -1)
	nRankDofs++;
142
143
    }
    
144
145
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
146
    void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
147
    {
148
      FUNCNAME("FeSpaceDofMap::insert()");
149
150
151
      
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
      
Thomas Witkowski's avatar
Thomas Witkowski committed
152
      dofMap[dof0].local = dof1;
Thomas Witkowski's avatar
Thomas Witkowski committed
153
      nLocalDofs++;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
154
      nonRankDofs.insert(dof0);
155
156
    }
    
157
    /// Checks if a given DOF is in the DOF mapping.
158
159
160
161
    bool isSet(DegreeOfFreedom dof)
    {
      return static_cast<bool>(dofMap.count(dof));
    }
162

163
164
165
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
    /// a DOF of the mapping (this is not checked here), otherwise the result is
    /// meaningsless.
166
167
168
169
    bool isRankDof(DegreeOfFreedom dof)
    {
      return !(static_cast<bool>(nonRankDofs.count(dof)));
    }
170
    
171
    /// Returns number of DOFs in the mapping.
172
173
174
175
176
    unsigned int size()
    {
      return dofMap.size();
    }
    
177
    /// Returns the raw data of the mapping.
178
    DofMap& getMap()
179
180
181
    {
      return dofMap;
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
182

183
184
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
185

186
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
187
188
189
190
191
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

192
193
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
194
    void setNonLocalDofs(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
195
    {
196
      hasNonLocalDofs = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
197
198
    }

199
    /// Informs the mapping whether a global index must be computed.
200
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
201
    {
202
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
203
204
    }

205
    /// Sets the DOF communicators.
206
    void setDofComm(DofComm &pSend, DofComm &pRecv)
Thomas Witkowski's avatar
Thomas Witkowski committed
207
    {
208
209
      sendDofs = &pSend;
      recvDofs = &pRecv;
Thomas Witkowski's avatar
Thomas Witkowski committed
210
211
    }

212
  private:
213
214
215
216
217
218
219
220
221
    /// Computes a global mapping from the local one.
    void computeGlobalMapping();

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
    void computeNonLocalIndices();

  private:
    /// MPI communicator object;
222
    MPI::Intracomm mpiComm;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
223

224
225
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
226

227
228
    /// The FE space this mapping belongs to. This is used only the get the
    /// correct DOF communicator in \ref sendDofs and \ref recvDofs.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
229
230
    const FiniteElemSpace *feSpace;

231
    /// Mapping data from DOF indices to local and global indices.
232
    DofMap dofMap;
233

234
    /// Set of all DOFs that are in mapping but are not owned by the rank.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
235
236
    std::set<DegreeOfFreedom> nonRankDofs;

237
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
238
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
239

240
241
242
243
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
244
245
246
  public:
    /// 
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
247
248
  };
  
249
250
251
252
253
254
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
255
  class ParallelDofMapping
256
257
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
258
    ParallelDofMapping() 
259
      : mpiComm(NULL),
260
261
	sendDofs(NULL),
	recvDofs(NULL),
262
	hasNonLocalDofs(false),
263
264
	feSpaces(0),
	nRankDofs(-1),
265
	nLocalDofs(-1),
266
267
268
	nOverallDofs(-1),
	rStartDofs(-1)
    {} 
269

270
271
272
273
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
274
275
276
277
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
278
279
280
281
282
     * \param[in]  needGlobalMapping  If true, the mapping computes also a global
     *                                index for the DOFs.
     * \param[in]  bNonLocalDofs      If true, at least one rank's mapping con-
     *                                taines DOFs that are not owend by the rank.
     */
283
    void init(MPI::Intracomm m,
284
	      vector<const FiniteElemSpace*> &fe,
285
	      vector<const FiniteElemSpace*> &uniqueFe,
286
287
	      bool needGlobalMapping,
	      bool bNonLocalDofs);
288
289
290
    
    /// Clear all data.
    void clear();
291
292
293
294
295
296
297

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
    void setDofComm(DofComm &pSend, DofComm &pRecv);

    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
298
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
299
      FUNCNAME("ParallelDofMapping::operator[]()");
300

301
302
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
303
304
305
306

      return data.find(feSpace)->second;
    }

307
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
308
309
310
311
312
313
314
    inline int getRankDofs()
    {
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");

      return nRankDofs;
    }

315
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
Thomas Witkowski's avatar
Thomas Witkowski committed
316
317
318
319
320
321
322
    inline int getLocalDofs()
    {
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");

      return nLocalDofs;
    }

323
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
324
325
326
327
328
329
330
    inline int getOverallDofs()
    {
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");

      return nOverallDofs;
    }

331
332
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
333
    inline int getStartDofs()
334
335
336
337
338
339
    {
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");

      return rStartDofs;
    }

340
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
341
    void update();
342

343
344
345
    /// Update the mapping.
    void update(vector<const FiniteElemSpace*>& feSpaces);

346
347
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
Thomas Witkowski's avatar
Thomas Witkowski committed
348
349
350
351
352
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
    {
      return dofToMatIndex.get(ithComponent, d);
    }

353
354
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
355
356
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
357
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
358
359
360
361
362
363
364

      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
	("Should not happen!\n");

      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
365
366
367
368
369
370
371
372
373
374
375
376
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
    int computeRankDofs();

    /// Compute \ref nLocalDofs.
    int computeLocalDofs();

    /// Compute \ref nOverallDofs.
    int computeOverallDofs();

    /// Compute \ref rStartDofs.
    int computeStartDofs();

    /// Compute local and global matrix indices.
    void computeMatIndex();

396
  private:
397
    /// MPI communicator object;
398
    MPI::Intracomm mpiComm;
399

400
401
    /// DOF communicators for all DOFs on interior boundaries.
    DofComm *sendDofs, *recvDofs;
402

403
404
405
406
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
    bool hasNonLocalDofs;
407

408
409
410
411
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
412
413
    vector<const FiniteElemSpace*> feSpaces;

414
415
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
416
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
417

418
419
420
421
422
423
424
425
426
427
428
    /// Number of DOFs owned by rank.
    int nRankDofs;

    /// Number of DOFs in rank's subdomain.
    int nLocalDofs;

    /// Number of global DOFs (this value is thus the same on all ranks).
    int nOverallDofs;

    /// Smallest global index of a DOF owned by the rank.
    int rStartDofs;
429
430
431
432

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
    DofToMatIndex dofToMatIndex;
433
434
435
436
437
  };

}

#endif