ParallelDomainProblem.h 11.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  crystal growth group                                                  ==
// ==                                                                        ==
// ==  Stiftung caesar                                                       ==
// ==  Ludwig-Erhard-Allee 2                                                 ==
// ==  53175 Bonn                                                            ==
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  http://www.caesar.de/cg/AMDiS                                         ==
// ==                                                                        ==
// ============================================================================

20
/** \file ParallelDomain.h */
21

22
23
#ifndef AMDIS_PARALLELDOMAIN_H
#define AMDIS_PARALLELDOMAIN_H
24
25

#include <map>
26
#include <set>
27
28
29
30
#include <vector>

#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
31
#include "FiniteElemSpace.h"
32
#include "AdaptInfo.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
33
#include "InteriorBoundary.h"
34
35
#include "AMDiS_fwd.h"

36
37
#include "petsc.h"
#include "petscsys.h"
38
#include "petscao.h"
39
40
41
42
43
44
#include "mpi.h"

namespace AMDiS {

  class ParMetisPartitioner;

45
  class ParallelDomainBase : public ProblemIterationInterface,
46
47
                                    public ProblemTimeInterface
  {
Thomas Witkowski's avatar
Thomas Witkowski committed
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  private:
    /// Defines type for a vector of DOFs.
    typedef std::vector<const DegreeOfFreedom*> DofContainer;

    /// Defines a mapping type from DOFs to rank numbers.
    typedef std::map<const DegreeOfFreedom*, int> DofToRank;

    /// Defines a mapping type from DOFs to a set of rank numbers.
    typedef std::map<const DegreeOfFreedom*, std::set<int> > DofToPartitions;

    /// Defines a mapping type from rank numbers to sets of DOFs.
    typedef std::map<int, DofContainer> RankToDofContainer;

    /// Defines a mapping type from DOF indices to DOF indices.
    typedef std::map<DegreeOfFreedom, DegreeOfFreedom> DofMapping;

    /// Defines a mapping type from DOF indices to boolean values.
    typedef std::map<DegreeOfFreedom, bool> DofToBool;

Thomas Witkowski's avatar
Thomas Witkowski committed
67
68
69
70
71
72
    /// Defines a mapping type from rank numbers to sets of coordinates.
    typedef std::map<int, std::vector<WorldVector<double> > > RankToCoords;

    /// Forward type (it maps rank numbers to the interior boundary objects).
    typedef InteriorBoundary::RankToBoundMap RankToBoundMap;

73
  public:
74
    ParallelDomainBase(const std::string& name,
Thomas Witkowski's avatar
Thomas Witkowski committed
75
76
77
78
		       ProblemIterationInterface *iterationIF,
		       ProblemTimeInterface *timeIF,
		       FiniteElemSpace *feSpace,
		       RefinementManager *refineManager);
79

80
    virtual ~ParallelDomainBase() {}
81

82
    virtual void initParallelization(AdaptInfo *adaptInfo);
83

84
85
86
87
88
89
90
91
92
    void exitParallelization(AdaptInfo *adaptInfo);

    /** \brief
     * Test, if the mesh consists of macro elements only. The mesh partitioning of
     * the parallelization works for macro meshes only and would fail, if the mesh
     * is already refined in some way. Therefore, this function will exit the program
     * if it finds a non macro element in the mesh.
     */
    void testForMacroMesh();
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

    /// Set for each element on the partitioning level the number of leaf elements.
    double setElemWeights(AdaptInfo *adaptInfo);

    void partitionMesh(AdaptInfo *adaptInfo);

    virtual void setTime(AdaptInfo *adaptInfo) {}

    virtual void initTimestep(AdaptInfo *adaptInfo) {}

    virtual void closeTimestep(AdaptInfo *adaptInfo) {}

    virtual void solveInitialProblem(AdaptInfo *adaptInfo) {}
  
    virtual void transferInitialSolution(AdaptInfo *adaptInfo) {}

109
110
    virtual void beginIteration(AdaptInfo *adaptInfo) 
    {
111
112
113
      iterationIF->beginIteration(adaptInfo);
    }

114
115
    virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION) 
    {
116
117
118
      ERROR_EXIT("Not implemented!\n");
      return 0;
    }
119

120
121
    virtual void endIteration(AdaptInfo *adaptInfo) 
    {
122
123
      iterationIF->endIteration(adaptInfo);
    }
124

125
126
127
128
    virtual int getNumProblems() 
    {
      return 0;
    }
129

130
131
    inline virtual const std::string& getName() 
    { 
132
133
134
      return name; 
    }

135
    /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
136
137
    int getNumberRankDOFs() 
    {
138
139
140
      return nRankDOFs;
    }

141
142
    void fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec);

143
144
    void solvePetscMatrix(DOFVector<double> *vec);

145
146
147
148
    virtual ProblemStatBase *getProblem(int number = 0) 
    {
      return NULL;
    }
149
150
151
152

    virtual void serialize(std::ostream&) {}

    virtual void deserialize(std::istream&) {}
153
154
155


  protected:
156
157
158
159
    /** \brief
     * Determine the interior boundaries, i.e. boundaries between ranks, and store
     * all information about them in \ref interiorBoundary.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
160
161
    void createInteriorBoundaryInfo(DofContainer& rankDOFs,
				    DofToRank& boundaryDOFs);
162
163
164
165

    /// Removes all macro elements from the mesh that are not part of ranks partition.
    void removeMacroElements();

166
167
168
169

    /** \brief
     * Creates from a macro mesh a correct local and global DOF index numbering.
     *
Thomas Witkowski's avatar
Thomas Witkowski committed
170
     * \param[out] rankDOFs      Returns all DOFs from the macro mesh, which are owned
171
     *                           by the rank after partitioning the macro mesh.
Thomas Witkowski's avatar
Thomas Witkowski committed
172
     * \param[out] boundaryDOFs  Returns all DOFs from the macro mesh, which lies at
173
174
     *                           an interior boundary of the rank. This object maps
     *                           each such DOF to the rank that owns this DOF.
Thomas Witkowski's avatar
Thomas Witkowski committed
175
176
     * \param[out] nRankDOFs     Number of DOFs owned by rank.
     * \param[out] nOverallDOFs  Number of all DOFs in macro mesh.
177
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
178
179
    void createLocalGlobalNumbering(DofContainer& rankDOFs,
				    DofToRank& boundaryDOFs,
180
181
182
183
				    int& nRankDOFs, 
				    int& nOverallDOFs);

    void updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs);
184

Thomas Witkowski's avatar
Thomas Witkowski committed
185
186
187
    void addAllVertexDOFs(Element *el, int ithEdge, DofContainer& dofs);

    void addAllEdgeDOFs(Element *el, int ithEdge, DofContainer& dofs);
Thomas Witkowski's avatar
Thomas Witkowski committed
188

189
190
    /** \brief
     * This function traverses the whole mesh, i.e. before it is really partitioned,
Thomas Witkowski's avatar
Thomas Witkowski committed
191
192
193
     * and collects information about which DOF corresponds to which rank. Can only
     * be used, if \ref partitionVec is set correctly. This is only the case, when
     * the macro mesh is partitioned.
194
     *
Thomas Witkowski's avatar
Thomas Witkowski committed
195
     * \param[out] partionDOFs   Stores to each DOF pointer the set of ranks the DOF is
196
     *                           part of.
Thomas Witkowski's avatar
Thomas Witkowski committed
197
198
     * \param[out] rankDOFs      Stores all rank DOFs.
     * \param[out] boundaryDOFs  Stores all DOFs in ranks partition that are on an 
199
200
     *                           interior boundary but correspond to another rank.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
201
202
203
    void createDOFMemberInfo(DofToPartitions& partitionDOFs,
			     DofContainer& rankDOFs,
			     DofToRank& boundaryDOFs);
Thomas Witkowski's avatar
Thomas Witkowski committed
204

Thomas Witkowski's avatar
Thomas Witkowski committed
205
206

    void DbgTestInteriorBoundary();
207
     
Thomas Witkowski's avatar
Thomas Witkowski committed
208
209
210
211
212
213
    /** \brief
     * This function is used for debugging only. It traverses all interior boundaries
     * and compares the dof indices on them with the dof indices of the boundarys
     * neighbours. The function fails, when dof indices on an interior boundary does
     * not fit together.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
214
    void DbgTestCommonDofs();
215

216
  protected:
217
218
219
220
221
222
    ///
    ProblemIterationInterface *iterationIF;

    ///
    ProblemTimeInterface *timeIF;

223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

    /** \brief
     * MPI communicator collected all processes, which should
     * be used for calculation. The Debug procces is not included
     * in this communicator.
     */
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
    std::string name;

239
240
241
    /// Finite element space of the problem.
    FiniteElemSpace *feSpace;

242
243
244
    /// Mesh of the problem.
    Mesh *mesh;

245
246
247
    /// Info level.
    int info;

248
249
250
    /// Refinement manager for the mesh.
    RefinementManager *refinementManager;

251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
    /// Pointer to the paritioner which is used to devide a mesh into partitions.
    ParMetisPartitioner *partitioner;

    /// Weights for the elements, i.e., the number of leaf elements within this element.
    std::map<int, double> elemWeights;

    /// Is true, if the mesh was not partitioned before, otherwise it's false.
    bool initialPartitionMesh;

    /** \brief
     * Stores to every coarse element index the number of the partition it 
     * corresponds to.
     */
    std::map<int, int> partitionVec;

    /** \brief
     * Stores an old partitioning of elements. To every element index the number
     * of the parition it corresponds to is stored.
     */
    std::map<int, int> oldPartitionVec;    
271

272
273
274
275
    Mat petscMatrix;

    Vec petscRhsVec;
    
276
    Vec petscSolVec;
277

278
279
    /// Number of DOFs in the rank mesh.
    int nRankDOFs;
Thomas Witkowski's avatar
Thomas Witkowski committed
280

281
282
283
284
    /** \brief
     * Set of all interior boundary DOFs in ranks partition. The object maps to
     * each such DOF to the number of the rank that owns this DOF.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
285
    DofToRank boundaryDOFs;
286

Thomas Witkowski's avatar
Thomas Witkowski committed
287
    /** \brief 
Thomas Witkowski's avatar
Thomas Witkowski committed
288
289
290
291
292
293
294
295
296
297
298
299
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are owned by the rank, i.e.,
     * the object gives for every neighbour rank i the boundaries this rank owns and 
     * shares with rank i.
     */
    InteriorBoundary myIntBoundary;
    
    /** \brief
     * Defines the interior boundaries of the domain that result from partitioning
     * the whole mesh. Contains only the boundaries, which are not owned by the rank,
     * i.e., the object gives for every neighbour rank i the boundaries that are
     * owned by rank i and are shared with this rank.
Thomas Witkowski's avatar
Thomas Witkowski committed
300
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
301
    InteriorBoundary otherIntBoundary;
Thomas Witkowski's avatar
Thomas Witkowski committed
302

303
304
305
306
    /** \brief
     * This map contains for each rank the list of dofs the current rank must send
     * to exchange solution dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
307
    RankToDofContainer sendDofs;
308
309
310
311
312

    /** \brief
     * This map contains for each rank the list of dofs from which the current rank 
     * must receive solution values of dofs at the interior boundaries.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
313
    RankToDofContainer recvDofs;
314
315

    /// Maps local to global dof indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
316
    DofMapping mapLocalGlobalDOFs;
317
318

    /// Maps global to local dof indices.
Thomas Witkowski's avatar
Thomas Witkowski committed
319
    DofMapping mapGlobalLocalDOFs;
320
321
322
323
324
325

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
     * owned by the rank. Otherwise, its an interior boundary DOF that is owned by
     * another rank.
     */
Thomas Witkowski's avatar
Thomas Witkowski committed
326
    DofToBool isRankDOF;
327
328
  };

329
  class ParallelDomainScal : public ParallelDomainBase
330
331
  {
  public:
332
    ParallelDomainScal(const std::string& name,
333
334
335
			      ProblemScal *problem,
			      ProblemInstatScal *problemInstat);

336
337
    void initParallelization(AdaptInfo *adaptInfo);

338
339
340
    virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION);

  protected:
341
342
    /// Starts the solution of the linear system using Petsc.
    void solve();
343

344
345
346
  protected:
    /// Pointer to the stationary problem.
    ProblemScal *probScal;
347
348
349
  };
}

350
#endif // AMDIS_PARALLELDOMAIN_H