Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

ParallelDofMapping.h 15.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ==  http://www.amdis-fem.org                                              ==
// ==                                                                        ==
// ============================================================================
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.



/** \file FeSpaceMapping.h */

23
#include <vector>
24
#include <map>
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
25 26
#include <set>
#include "parallel/DofComm.h"
27
#include "parallel/MeshLevelData.h"
28 29
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
30
#include "parallel/StdMpi.h"
31

32 33
#include <petscis.h>

34 35 36 37
#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H

namespace AMDiS {
Thomas Witkowski's avatar
Thomas Witkowski committed
38

39 40
  using namespace std;

41
  /** \brief
42 43 44
   * Is used to store matrix indices to all DOFs in rank's subdomain. Thus, the
   * class defines a mapping from component number and DOF index to a global
   * matrix index. This class does not calculate the indices by itself!
45 46 47 48 49 50 51 52 53 54 55 56
   */
  class DofToMatIndex
  {
  public:
    DofToMatIndex() {}

    /// Reset the data structure.
    inline void clear()
    {
      data.clear();
    }

57 58 59 60 61 62 63
    /** Add a new mapping for a given DOF.
     * 
     * \param[in]   component       Component number for which the mapping 
     *                              is defined.
     * \param[in]   dof             DOF index
     * \param[in]   globalMatIndex  Global matrix index.
     */
64 65 66 67 68
    inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
    {
      data[component][dof] = globalMatIndex;
    }

69
    /// Maps a global DOF index to the global matrix index for a specific 
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
    /// system component number.
    inline int get(int component, DegreeOfFreedom dof)
    {
      FUNCNAME("DofToMatIndex::get()");

      TEST_EXIT_DBG(data.count(component))
	("No mapping data for component %d available!\n", component);

      TEST_EXIT_DBG(data[component].count(dof))
	("Mapping for DOF %d in component %d does not exists!\n",
	 dof, component);

      return data[component][dof];
    }

85 86 87 88 89
    /// Returns for a given matrix index the component and (local or global) DOF
    /// index. As the data structure is not made for this kind of reverse
    /// search, this is very slow and should be only used for debugging.
    void getReverse(int rowIndex, int &component, int &dofIndex);

90 91 92 93 94 95 96
  private:
    /// The mapping data. For each system component there is a specific map that
    /// maps global DOF indices to global matrix indices.
    map<int, map<DegreeOfFreedom, int> > data;
  };


97 98 99 100 101 102
  /**
   * This class defines the parallel mapping of DOFs for one FE space. It is used
   * by the class \ref ParallelDofMapping to specifiy the mapping for a set of 
   * FE spaces.
   */
  class FeSpaceDofMap
103 104
  {
  public:
105 106
    /// This constructor exists only to create std::map of this class and make
    /// use of the operator [] for read access. Should never be called.
107
    FeSpaceDofMap() 
108 109 110
    {
      ERROR_EXIT("Should not be called!\n");
    }
111 112
     
    /// This is the only valid constructur to be used. 
113 114
    FeSpaceDofMap(MeshLevelData* ld)
      : levelData(ld),
115
	dofComm(NULL),
116
	feSpace(NULL),
Thomas Witkowski's avatar
Thomas Witkowski committed
117
	needGlobalMapping(false),
118
	isNonLocal(false)
119 120 121
    {
      clear();
    }
122 123

    /// Clears all data of the mapping.
124
    void clear();
125

126 127
    /// Maps a DOF index to both, the local and global index of the mapping. The
    /// global index must not be set.
Thomas Witkowski's avatar
Thomas Witkowski committed
128
    MultiIndex& operator[](DegreeOfFreedom d)
129
    {
130
      TEST_EXIT_DBG(dofMap.count(d))("DOF %d is not in map!\n", d);
131

132
      return dofMap[d];
133
    }
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152

    /** \brief
     * Searchs the map for a given DOF. It does not fail, if the DOF is not 
     * mapped by this mapping. In this case, it returns false. If the DOF is 
     * mapped, the result is stored and the function returns true.
     *
     * \param[in]    dof     DOF index for which a mapping is searched.
     * \param[out]   index   In the case that the DOF is mapped, the result
     *                       is stored here.
     */
    inline bool find(DegreeOfFreedom dof, MultiIndex& index)
    {
      DofMap::iterator it = dofMap.find(dof);
      if (it == dofMap.end())
	return false;

      index = it->second;
      return true;
    }
153
    
154 155
    /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
    /// the rank.
156
    void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
157
    {
158
      FUNCNAME("FeSpaceDofMap::insertRankDof()");
159
      
160
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
161
      
162 163
      dofMap[dof0].local = dof1;
      nLocalDofs++;
Thomas Witkowski's avatar
Thomas Witkowski committed
164
      if (dof1 != -1)
165
	nRankDofs++;
166 167
    }
    
168 169
    /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
    /// but is owned by a different rank, thus it is part of an interior boundary.
170
    void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
171
    {
172
      FUNCNAME("FeSpaceDofMap::insertNonRankDof()");
173
      
174
      TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
175
      
176 177 178
      dofMap[dof0].local = dof1;
      nLocalDofs++;
      nonRankDofs.insert(dof0);
179 180
    }
    
181
    /// Checks if a given DOF is in the DOF mapping.
182
    bool isSet(DegreeOfFreedom dof)
183
    {
184
      return static_cast<bool>(dofMap.count(dof));
185
    }
186

Thomas Witkowski's avatar
Blbu  
Thomas Witkowski committed
187 188 189
    /// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF
    /// must be a DOF of the mapping (this is not checked here), otherwise the
    /// result is meaningsless.
190
    bool isRankDof(DegreeOfFreedom dof)
191
    {
192
      return !(static_cast<bool>(nonRankDofs.count(dof)));
193
    }
Thomas Witkowski's avatar
Blbu  
Thomas Witkowski committed
194 195 196 197 198

    bool isRankGlobalDof(int dof)
    {
      return (dof >= rStartDofs && dof < rStartDofs + nRankDofs);
    }
199
    
200
    /// Returns number of DOFs in the mapping.
201
    unsigned int size()
202
    {
203
      return dofMap.size();
204 205
    }
    
206
    /// Returns the raw data of the mapping.
207
    DofMap& getMap()
208
    {
209
      return dofMap;
210
    }
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
211

212 213 214 215 216 217 218 219 220 221
    DofMap::iterator begin()
    {
      return dofMap.begin();
    }

    DofMap::iterator end()
    {
      return dofMap.end();
    }

222 223
    /// Recomputes the mapping.
    void update();
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
224

225
    /// Sets the FE space this mapping corresponds to.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
226 227 228 229 230
    void setFeSpace(const FiniteElemSpace *fe)
    {
      feSpace = fe;
    }

231 232
    /// Informs the mapping whether the mapping will include DOFs that are not
    /// owned by the rank.
233
    void setNonLocal(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
234
    {
235
      isNonLocal = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
236 237
    }

238
    /// Informs the mapping whether a global index must be computed.
239
    void setNeedGlobalMapping(bool b)
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
240
    {
241
      needGlobalMapping = b;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
242 243
    }

244
    /// Sets the DOF communicator.
245
    void setDofComm(DofComm &dc)
Thomas Witkowski's avatar
Thomas Witkowski committed
246
    {
247
      dofComm = &dc;
Thomas Witkowski's avatar
Thomas Witkowski committed
248 249
    }

250 251 252 253 254 255
    void setMpiComm(MPI::Intracomm &m, int l)
    {
      mpiComm = m;
      meshLevel = l;
    }

256
  private:
257
    /// Computes a global mapping from the local one.
258
    void computeGlobalMapping();
259 260 261

    /// Computes the global indices of all DOFs in the mapping that are not owned
    /// by the rank.
262
    void computeNonLocalIndices();
263 264

  private:
265
    MeshLevelData *levelData;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
266

267 268
    /// DOF communicator for all DOFs on interior boundaries.
    DofComm *dofComm;
269

270 271 272 273
    MPI::Intracomm mpiComm;

    int meshLevel;

274
    /// The FE space this mapping belongs to. This is used only the get the
275
    /// correct DOF communicator in \ref dofComm.
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
276 277
    const FiniteElemSpace *feSpace;

278
    /// Mapping data from DOF indices to local and global indices.
279
    DofMap dofMap;
280

281
    /// Set of all DOFs that are in mapping but are not owned by the rank.
282
    std::set<DegreeOfFreedom> nonRankDofs;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
283

284
    /// If true, a global index mapping will be computed for all DOFs.
Thomas Witkowski's avatar
Thomas Witkowski committed
285
    bool needGlobalMapping;
Thomas Witkowski's avatar
FETI-DP  
Thomas Witkowski committed
286

287 288 289
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
290
    bool isNonLocal;
291

292 293
  public:
    /// 
294
    int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
295 296
  };
  
297 298 299 300 301 302
  
  /**
   * Implements the mapping from sets of distributed DOF indices to local and
   * global indices. The mapping works for a given set of FE spaces. Furthermore,
   * this class may compute the matrix indices of the set of DOF indices.
   */
Thomas Witkowski's avatar
Thomas Witkowski committed
303
  class ParallelDofMapping
304 305
  {
  public:
Thomas Witkowski's avatar
Thomas Witkowski committed
306
    ParallelDofMapping() 
307
      : levelData(NULL),
308
	dofComm(NULL),
309
	isNonLocal(true),
310
	needMatIndexFromGlobal(false),
311
	feSpaces(0),
312 313 314 315 316
	nRankDofs(1),
	nLocalDofs(1),
	nOverallDofs(1),
	rStartDofs(1)
    {
317 318 319 320
      nRankDofs = -1;
      nLocalDofs = -1;
      nOverallDofs = -1;
      rStartDofs = -1;
321
    } 
322

323 324 325 326
    /** \brief Initialize the parallel DOF mapping.
     *
     * \param[in]  m                  MPI communicator.
     * \param[in]  fe                 The FE spaces of all components of the 
327 328 329 330
     *                                PDE to be solved.
     * \param[in]  uniqueFe           Unique list of FE spaces. Thus, two
     *                                arbitrary elements of this list are always
     *                                different.
331
     * \param[in]  isNonLocal         If true, at least one rank's mapping con-
332 333
     *                                taines DOFs that are not owend by the rank.
     */
334
    void init(MeshLevelData& mld,
335
	      vector<const FiniteElemSpace*> &fe,
336
	      vector<const FiniteElemSpace*> &uniqueFe,
337 338 339 340 341 342 343 344 345 346 347 348
	      bool isNonLocal = true);

    /// In the case of having only one FE space, this init function can be used.
    void init(MeshLevelData& mld,
	      const FiniteElemSpace *feSpace,
	      bool isNonLocal = true)
    {
      vector<const FiniteElemSpace*> feSpaces;
      feSpaces.push_back(feSpace);
      init(mld, feSpaces, feSpaces, isNonLocal);
    }

349
    void setMpiComm(MPI::Intracomm &m, int l);
350 351 352
    
    /// Clear all data.
    void clear();
353 354 355

    /// Set the DOF communicator objects that are required to exchange information
    /// about DOFs that are on interior boundaries.
356
    void setDofComm(DofComm &dofComm);
357

358 359 360
    /// Returns the DOF communicator.
    DofComm& getDofComm()
    {
361 362 363
      FUNCNAME("ParallelDofMapping::getDofComm()");

      TEST_EXIT_DBG(dofComm)("No DOF communicator object defined!\n");
364 365 366 367

      return *dofComm;
    }

368 369 370
    /// Changes the computation of matrix indices based of either local or
    /// global DOF indices, see \ref needMatIndexFromGlobal
    void setComputeMatIndex(bool global)
371 372 373 374
    {
      needMatIndexFromGlobal = global;
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
375 376 377 378 379
    inline bool isMatIndexFromGlobal()
    {
      return needMatIndexFromGlobal;
    }

380 381
    /// Access the DOF mapping for a given FE space.
    inline FeSpaceDofMap& operator[](const FiniteElemSpace* feSpace)
382
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
383
      FUNCNAME("ParallelDofMapping::operator[]()");
384

385 386
      TEST_EXIT_DBG(data.count(feSpace))
	("No data for FE space at address %p!\n", feSpace);
387 388 389 390

      return data.find(feSpace)->second;
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
391 392 393 394 395
    inline bool isDefinedFor(const FiniteElemSpace* feSpace) const
    {
      return static_cast<bool>(data.count(feSpace));
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
396 397 398 399 400 401
    /// Returns the number of solution components the mapping is defined on.
    inline int getNumberOfComponents() const
    {
      return static_cast<int>(feSpaces.size());
    }

402
    /// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
403
    inline int getRankDofs()
404
    {
405
      TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");
406

407
      return nRankDofs;
408 409
    }

410
    /// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
411
    inline int getLocalDofs()
Thomas Witkowski's avatar
Thomas Witkowski committed
412
    {
413
      TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");
Thomas Witkowski's avatar
Thomas Witkowski committed
414

415
      return nLocalDofs;
Thomas Witkowski's avatar
Thomas Witkowski committed
416 417
    }

418
    /// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
419
    inline int getOverallDofs()
420
    {
421
      TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");
422

423
      return nOverallDofs;
424 425
    }

426 427
    /// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
    /// owned by the rank.
428
    inline int getStartDofs()
429
    {
430
      TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");
431

432
      return rStartDofs;
433 434
    }

435
    /// Update the mapping.
Thomas Witkowski's avatar
Thomas Witkowski committed
436
    void update();
437

438 439 440
    /// Update the mapping.
    void update(vector<const FiniteElemSpace*>& feSpaces);

441 442
    /// Returns the global matrix index of a given DOF for a given 
    /// component number.
443
    inline int getMatIndex(int ithComponent, DegreeOfFreedom d)
Thomas Witkowski's avatar
Thomas Witkowski committed
444
    {
445
      return dofToMatIndex.get(ithComponent, d);
Thomas Witkowski's avatar
Thomas Witkowski committed
446 447
    }

448 449 450 451 452 453 454
    /// Returns the component number and local/global DOF index for a given
    /// matrix row index. Should be used for debugging only!
    inline void getReverseMatIndex(int index, int &component, int &dofIndex)
    {
      dofToMatIndex.getReverse(index, component, dofIndex);
    }

455 456
    /// Returns the local matrix index of a given DOF for a given 
    /// component number.
457
    inline int getLocalMatIndex(int ithComponent, DegreeOfFreedom d)
458
    {
Thomas Witkowski's avatar
Thomas Witkowski committed
459
      FUNCNAME("ParallelDofMapping::getLocalMatIndex()");
460

461
      TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
462 463
	("Should not happen!\n");

464
      return dofToMatIndex.get(ithComponent, d) - rStartDofs;
465 466
    }

Thomas Witkowski's avatar
Thomas Witkowski committed
467 468 469 470 471 472 473 474 475 476 477 478
    // Writes all data of this object to an output stream.
    void serialize(ostream &out)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

    // Reads the object data from an input stream.
    void deserialize(istream &in)
    {
      ERROR_EXIT("MUST BE IMPLEMENTED!\n");
    }

479
    /// Compute local and global matrix indices.
480
    void computeMatIndex(bool globalIndex);
481

482 483 484 485
    void createIndexSet(IS &is, 
			int firstComponent, 
			int nComponents);

486 487 488 489 490
  protected:
    /// Insert a new FE space DOF mapping for a given FE space.
    void addFeSpace(const FiniteElemSpace* feSpace);

    /// Compute \ref nRankDofs.
491
    int computeRankDofs();
492 493

    /// Compute \ref nLocalDofs.
494
    int computeLocalDofs();
495 496

    /// Compute \ref nOverallDofs.
497
    int computeOverallDofs();
498 499

    /// Compute \ref rStartDofs.
500
    int computeStartDofs();
501

502
  private:
503 504 505 506
    MPI::Intracomm mpiComm;

    int meshLevel;

507
    MeshLevelData *levelData;
508

509 510
    /// DOF communicator for all DOFs on interior boundaries.
    DofComm *dofComm;
511

512 513 514
    /// Is true if there are DOFs in at least one subdomain that are not owned
    /// by the rank. If the value is false, each rank contains only DOFs that
    /// are also owned by this rank.
515
    bool isNonLocal;
516 517 518 519 520

    /// If matrix indices should be computed, this variable defines if the
    /// mapping from DOF indices to matrix row indices is defined on local
    /// or global DOF indices. If true, the mapping is to specify and to use
    /// on global ones, otherwise on local DOF indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
521 522 523 524 525
    /// In most scenarios the mapping stored on local DOF indices is what we
    /// want to have. Only when periodic boundary conditions are used together
    /// with some global matrix approache, the matrix indices must be stored
    /// also for DOFs that are not part of the local subdomain. Thus, the
    /// mapping will be stored on global DOF indices.
526 527
    bool needMatIndexFromGlobal;

528 529 530 531
    /// Maps from FE space pointers to DOF mappings.
    map<const FiniteElemSpace*, FeSpaceDofMap> data;

    /// The FE spaces for all components.
532 533
    vector<const FiniteElemSpace*> feSpaces;

534 535
    /// The set of all FE spaces. It uniquly contains all different FE spaces
    /// from \ref feSpaces.
536
    vector<const FiniteElemSpace*> feSpacesUnique;
Thomas Witkowski's avatar
Thomas Witkowski committed
537

538
    /// Number of DOFs owned by rank.
539
    int nRankDofs;
540 541

    /// Number of DOFs in rank's subdomain.
542
    int nLocalDofs;
543 544

    /// Number of global DOFs (this value is thus the same on all ranks).
545
    int nOverallDofs;
546 547

    /// Smallest global index of a DOF owned by the rank.
548
    int rStartDofs;
549 550 551

    /// Mapping from global DOF indices to global matrix indices under 
    /// consideration of possibly multiple components.
552
    DofToMatIndex dofToMatIndex;
553 554 555 556 557
  };

}

#endif