ParallelDomainProblem.h 12.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  crystal growth group                                                  ==
// ==                                                                        ==
// ==  Stiftung caesar                                                       ==
// ==  Ludwig-Erhard-Allee 2                                                 ==
// ==  53175 Bonn                                                            ==
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  http://www.caesar.de/cg/AMDiS                                         ==
// ==                                                                        ==
// ============================================================================

20
/** \file ParallelDomain.h */
21

22
23
#ifndef AMDIS_PARALLELDOMAIN_H
#define AMDIS_PARALLELDOMAIN_H
24
25

#include <map>
26
#include <set>
27
28
29
30
#include <vector>

#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
31
#include "FiniteElemSpace.h"
32
#include "AdaptInfo.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
33
#include "InteriorBoundary.h"
34
35
#include "AMDiS_fwd.h"

36
37
#include "petsc.h"
#include "petscsys.h"
38
#include "petscao.h"
39
40
41
42
43
44
#include "mpi.h"

namespace AMDiS {

  class ParMetisPartitioner;

45
  class ParallelDomainBase : public ProblemIterationInterface,
46
47
                                    public ProblemTimeInterface
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  private:
    /// Defines type for a vector of DOFs.
    typedef std::vector<const DegreeOfFreedom*> DofContainer;

    /// Defines a mapping type from DOFs to rank numbers.
    typedef std::map<const DegreeOfFreedom*, int> DofToRank;

    /// Defines a mapping type from DOFs to a set of rank numbers.
    typedef std::map<const DegreeOfFreedom*, std::set<int> > DofToPartitions;

    /// Defines a mapping type from rank numbers to sets of DOFs.
    typedef std::map<int, DofContainer> RankToDofContainer;

    /// Defines a mapping type from DOF indices to DOF indices.
    typedef std::map<DegreeOfFreedom, DegreeOfFreedom> DofMapping;

    /// Defines a mapping type from DOF indices to boolean values.
    typedef std::map<DegreeOfFreedom, bool> DofToBool;

Thomas Witkowski's avatar
Thomas Witkowski committed
67
68
69
70
71
72
    /// Defines a mapping type from rank numbers to sets of coordinates.
    typedef std::map<int, std::vector<WorldVector<double> > > RankToCoords;

    /// Forward type (it maps rank numbers to the interior boundary objects).
    typedef InteriorBoundary::RankToBoundMap RankToBoundMap;

Thomas Witkowski's avatar
Thomas Witkowski committed
73
74
    typedef std::map<int, DofContainer> ElementIdxToDofs;

Thomas Witkowski's avatar
Thomas Witkowski committed
75
76
    typedef std::map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;

77
  public:
78
    ParallelDomainBase(const std::string& name,
Thomas Witkowski's avatar
Thomas Witkowski committed
79
80
81
82
		       ProblemIterationInterface *iterationIF,
		       ProblemTimeInterface *timeIF,
		       FiniteElemSpace *feSpace,
		       RefinementManager *refineManager);
83

84
    virtual ~ParallelDomainBase() {}
85

86
    virtual void initParallelization(AdaptInfo *adaptInfo);
87

88
89
90
91
92
93
94
95
96
    void exitParallelization(AdaptInfo *adaptInfo);

    /** \brief
     * Test, if the mesh consists of macro elements only. The mesh partitioning of
     * the parallelization works for macro meshes only and would fail, if the mesh
     * is already refined in some way. Therefore, this function will exit the program
     * if it finds a non macro element in the mesh.
     */
    void testForMacroMesh();
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

    /// Set for each element on the partitioning level the number of leaf elements.
    double setElemWeights(AdaptInfo *adaptInfo);

    void partitionMesh(AdaptInfo *adaptInfo);

    virtual void setTime(AdaptInfo *adaptInfo) {}

    virtual void initTimestep(AdaptInfo *adaptInfo) {}

    virtual void closeTimestep(AdaptInfo *adaptInfo) {}

    virtual void solveInitialProblem(AdaptInfo *adaptInfo) {}
  
    virtual void transferInitialSolution(AdaptInfo *adaptInfo) {}

113
114
    virtual void beginIteration(AdaptInfo *adaptInfo) 
    {
115
116
117
      iterationIF->beginIteration(adaptInfo);
    }

118
119
    virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION) 
    {
120
121
122
      ERROR_EXIT("Not implemented!\n");
      return 0;
    }
123

124
125
    virtual void endIteration(AdaptInfo *adaptInfo) 
    {
126
127
      iterationIF->endIteration(adaptInfo);
    }
128

129
130
131
132
    virtual int getNumProblems() 
    {
      return 0;
    }
133

134
135
    inline virtual const std::string& getName() 
    { 
136
137
138
      return name; 
    }

139
    /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
140
141
    int getNumberRankDOFs() 
    {
142
143
144
      return nRankDOFs;
    }

145
146
    void fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec);

147
148
    void solvePetscMatrix(DOFVector<double> *vec);

149
150
151
152
    virtual ProblemStatBase *getProblem(int number = 0) 
    {
      return NULL;
    }
153
154
155
156

    virtual void serialize(std::ostream&) {}

    virtual void deserialize(std::istream&) {}
157
158
159


  protected:
160
161
162
163
    /** \brief
     * Determine the interior boundaries, i.e. boundaries between ranks, and store
     * all information about them in \ref interiorBoundary.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
164
165
    void createInteriorBoundaryInfo(DofContainer& rankDOFs,
				    DofToRank& boundaryDOFs);
166
167
168
169

    /// Removes all macro elements from the mesh that are not part of ranks partition.
    void removeMacroElements();

170
171
172
173

    /** \brief
     * Creates from a macro mesh a correct local and global DOF index numbering.
     *
Thomas Witkowski's avatar
Thomas Witkowski committed
174
     * \param[out] rankDOFs      Returns all DOFs from the macro mesh, which are owned
175
     *                           by the rank after partitioning the macro mesh.
Thomas Witkowski's avatar
Thomas Witkowski committed
176
     * \param[out] boundaryDOFs  Returns all DOFs from the macro mesh, which lies at
177
178
     *                           an interior boundary of the rank. This object maps
     *                           each such DOF to the rank that owns this DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
179
180
     * \param[out] nRankDOFs     Number of DOFs owned by rank.
     * \param[out] nOverallDOFs  Number of all DOFs in macro mesh.
181
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
182
183
    void createLocalGlobalNumbering(DofContainer& rankDOFs,
				    DofToRank& boundaryDOFs,
184
185
186
187
				    int& nRankDOFs, 
				    int& nOverallDOFs);

    void updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs);
188

Thomas Witkowski's avatar
Thomas Witkowski committed
189
190
191
    void addAllVertexDOFs(Element *el, int ithEdge, DofContainer& dofs);

    void addAllEdgeDOFs(Element *el, int ithEdge, DofContainer& dofs);
Thomas Witkowski's avatar
Thomas Witkowski committed
192

193
194
    /** \brief
     * This function traverses the whole mesh, i.e. before it is really partitioned,
Thomas Witkowski's avatar
Thomas Witkowski committed
195
196
197
     * and collects information about which DOF corresponds to which rank. Can only
     * be used, if \ref partitionVec is set correctly. This is only the case, when
     * the macro mesh is partitioned.
198
     *
Thomas Witkowski's avatar
Thomas Witkowski committed
199
     * \param[out] partionDOFs   Stores to each DOF pointer the set of ranks the DOF is
200
     *                           part of.
Thomas Witkowski's avatar
Thomas Witkowski committed
201
202
     * \param[out] rankDOFs      Stores all rank DOFs.
     * \param[out] boundaryDOFs  Stores all DOFs in ranks partition that are on an 
203
204
     *                           interior boundary but correspond to another rank.
     */
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
205
206
207
208
    void createDOFMemberInfo(DofToPartitions& partitionDofs,
			     DofContainer& rankOwnedDofs,
			     DofContainer& rankAllDofs,
			     DofToRank& boundaryDofs);
Thomas Witkowski's avatar
Thomas Witkowski committed
209

Thomas Witkowski's avatar
Thomas Witkowski committed
210
211
212
    void DbgCreateElementMap(ElementIdxToDofs &elMap);
    
    void DbgTestElementMap(ElementIdxToDofs &elMap);
Thomas Witkowski's avatar
Thomas Witkowski committed
213
214

    void DbgTestInteriorBoundary();
215
     
Thomas Witkowski's avatar
Thomas Witkowski committed
216
217
218
219
220
221
    /** \brief
     * This function is used for debugging only. It traverses all interior boundaries
     * and compares the dof indices on them with the dof indices of the boundarys
     * neighbours. The function fails, when dof indices on an interior boundary does
     * not fit together.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
222
    void DbgTestCommonDofs();
223

Thomas Witkowski's avatar
Thomas Witkowski committed
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
    inline void orderDOFs(const DegreeOfFreedom* dof1,
			  const DegreeOfFreedom* dof2,
			  const DegreeOfFreedom* dof3,
			  DofContainer &vec)
    {
      vec.resize(3);

      if (*dof1 < *dof2 && *dof1 < *dof3)
	vec[0] = dof1;
      else if (*dof2 < *dof1 && *dof2 < *dof3)
	vec[0] = dof2;
      else 
	vec[0] = dof3;

      if (*dof1 > *dof2 && *dof1 > *dof3)
	vec[2] = dof1;
      else if (*dof2 > *dof1 && *dof2 > *dof3)
	vec[2] = dof2;
      else 
	vec[2] = dof3;

      if (dof1 != vec[0] && dof1 != vec[2]) 
	vec[1] = dof1;
      else if (dof2 != vec[0] && dof2 != vec[2]) 
	vec[1] = dof2;
      else
	vec[1] = dof3;
    }

253
  protected:
254
255
256
257
258
259
    ///
    ProblemIterationInterface *iterationIF;

    ///
    ProblemTimeInterface *timeIF;

260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

    /** \brief
     * MPI communicator collected all processes, which should
     * be used for calculation. The Debug procces is not included
     * in this communicator.
     */
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
    std::string name;

276
277
278
    /// Finite element space of the problem.
    FiniteElemSpace *feSpace;

279
280
281
    /// Mesh of the problem.
    Mesh *mesh;

282
283
284
    /// Info level.
    int info;

285
286
287
    /// Refinement manager for the mesh.
    RefinementManager *refinementManager;

288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
    /// Pointer to the paritioner which is used to devide a mesh into partitions.
    ParMetisPartitioner *partitioner;

    /// Weights for the elements, i.e., the number of leaf elements within this element.
    std::map<int, double> elemWeights;

    /// Is true, if the mesh was not partitioned before, otherwise it's false.
    bool initialPartitionMesh;

    /** \brief
     * Stores to every coarse element index the number of the partition it 
     * corresponds to.
     */
    std::map<int, int> partitionVec;

    /** \brief
     * Stores an old partitioning of elements. To every element index the number
     * of the parition it corresponds to is stored.
     */
    std::map<int, int> oldPartitionVec;    
308

309
310
311
312
    Mat petscMatrix;

    Vec petscRhsVec;
    
313
    Vec petscSolVec;
314

315
316
    /// Number of DOFs in the rank mesh.
    int nRankDOFs;
Thomas Witkowski's avatar
Thomas Witkowski committed
317

318
319
320
321
    /** \brief
     * Set of all interior boundary DOFs in ranks partition. The object maps to
     * each such DOF to the number of the rank that owns this DOF.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
322
    DofToRank boundaryDOFs;
323

Thomas Witkowski's avatar
Thomas Witkowski committed
324
    /** \brief 
Thomas Witkowski's avatar
Thomas Witkowski committed
325
326
327
328
329
330
331
332
333
334
335
336
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,
     * the object gives for every neighbour rank i the boundaries this rank owns and 
     * shares with rank i.
     */
    InteriorBoundary myIntBoundary;
    
    /** \brief
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are not owned by the rank,
     * i.e., the object gives for every neighbour rank i the boundaries that are
     * owned by rank i and are shared with this rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
337
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
338
    InteriorBoundary otherIntBoundary;
Thomas Witkowski's avatar
Thomas Witkowski committed
339

340
341
342
343
    /** \brief
     * This map contains for each rank the list of dofs the current rank must send
     * to exchange solution dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
344
    RankToDofContainer sendDofs;
345
346
347
348
349

    /** \brief
     * This map contains for each rank the list of dofs from which the current rank 
     * must receive solution values of dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
350
    RankToDofContainer recvDofs;
351
352

    /// Maps local to global dof indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
353
    DofMapping mapLocalGlobalDOFs;
354
355

    /// Maps global to local dof indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
356
    DofMapping mapGlobalLocalDOFs;
357
358
359
360
361
362

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
     * owned by the rank. Otherwise, its an interior boundary DOF that is owned by
     * another rank.
     */
Thomas Witkowski's avatar
n    
Thomas Witkowski committed
363
364
365
    DofToBool isRankDof;

    int rstart;
Thomas Witkowski's avatar
Thomas Witkowski committed
366
367

    DofIndexMap rankDofsNewLocalIndex, rankDofsNewGlobalIndex;
368
369
  };

Thomas Witkowski's avatar
Thomas Witkowski committed
370
371
372
373
374
  bool cmpDofsByValue(const DegreeOfFreedom* dof1, const DegreeOfFreedom* dof2)
  {
    return (*dof1 < *dof2);
  }

375
  class ParallelDomainScal : public ParallelDomainBase
376
377
  {
  public:
378
    ParallelDomainScal(const std::string& name,
379
380
381
			      ProblemScal *problem,
			      ProblemInstatScal *problemInstat);

382
383
    void initParallelization(AdaptInfo *adaptInfo);

384
385
386
    virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION);

  protected:
387
388
    /// Starts the solution of the linear system using Petsc.
    void solve();
389

390
391
392
  protected:
    /// Pointer to the stationary problem.
    ProblemScal *probScal;
393
394
395
  };
}

396
#endif // AMDIS_PARALLELDOMAIN_H