ParallelDomainProblem.h 8.29 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
// ============================================================================
// ==                                                                        ==
// == AMDiS - Adaptive multidimensional simulations                          ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  crystal growth group                                                  ==
// ==                                                                        ==
// ==  Stiftung caesar                                                       ==
// ==  Ludwig-Erhard-Allee 2                                                 ==
// ==  53175 Bonn                                                            ==
// ==  germany                                                               ==
// ==                                                                        ==
// ============================================================================
// ==                                                                        ==
// ==  http://www.caesar.de/cg/AMDiS                                         ==
// ==                                                                        ==
// ============================================================================

/** \file ParallelDomainProblem.h */

#ifndef AMDIS_PARALLELDOMAINPROBLEM_H
#define AMDIS_PARALLELDOMAINPROBLEM_H

#include <map>
26
#include <set>
27
28
29
30
#include <vector>

#include "ProblemTimeInterface.h"
#include "ProblemIterationInterface.h"
31
#include "FiniteElemSpace.h"
32
#include "AdaptInfo.h"
Thomas Witkowski's avatar
Thomas Witkowski committed
33
#include "InteriorBoundary.h"
34
35
#include "AMDiS_fwd.h"

36
37
#include "petsc.h"
#include "petscsys.h"
38
#include "petscao.h"
39
40
41
42
43
44
45
46
47
48
49
50
51
#include "mpi.h"

namespace AMDiS {

  class ParMetisPartitioner;

  class ParallelDomainProblemBase : public ProblemIterationInterface,
                                    public ProblemTimeInterface
  {
  public:
    ParallelDomainProblemBase(const std::string& name,
			      ProblemIterationInterface *iterationIF,
			      ProblemTimeInterface *timeIF,
52
53
			      FiniteElemSpace *feSpace,
			      RefinementManager *refineManager);
54
55
56

    virtual ~ParallelDomainProblemBase() {}

57
    virtual void initParallelization(AdaptInfo *adaptInfo);
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

    void exitParallelization(AdaptInfo *adaptInfo);    

    /// Set for each element on the partitioning level the number of leaf elements.
    double setElemWeights(AdaptInfo *adaptInfo);

    void partitionMesh(AdaptInfo *adaptInfo);

    virtual void setTime(AdaptInfo *adaptInfo) {}

    virtual void initTimestep(AdaptInfo *adaptInfo) {}

    virtual void closeTimestep(AdaptInfo *adaptInfo) {}

    virtual void solveInitialProblem(AdaptInfo *adaptInfo) {}
  
    virtual void transferInitialSolution(AdaptInfo *adaptInfo) {}

76
77
    virtual void beginIteration(AdaptInfo *adaptInfo) 
    {
78
79
80
      iterationIF->beginIteration(adaptInfo);
    }

81
82
    virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION) 
    {
83
84
85
      ERROR_EXIT("Not implemented!\n");
      return 0;
    }
86

87
88
    virtual void endIteration(AdaptInfo *adaptInfo) 
    {
89
90
      iterationIF->endIteration(adaptInfo);
    }
91

92
93
94
95
    virtual int getNumProblems() 
    {
      return 0;
    }
96

97
98
    inline virtual const std::string& getName() 
    { 
99
100
101
      return name; 
    }

102
    /// Returns \ref nRankDOFs, the number of DOFs in the rank mesh.
103
104
    int getNumberRankDOFs() 
    {
105
106
107
      return nRankDOFs;
    }

108
109
    void fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec);

110
111
    void solvePetscMatrix(DOFVector<double> *vec);

112
113
114
115
    virtual ProblemStatBase *getProblem(int number = 0) 
    {
      return NULL;
    }
116
117
118
119

    virtual void serialize(std::ostream&) {}

    virtual void deserialize(std::istream&) {}
120
121
122


  protected:
123
124
125
126
    /** \brief
     * Determine the interior boundaries, i.e. boundaries between ranks, and store
     * all information about them in \ref interiorBoundary.
     */
127
128
    void createInteriorBoundaryInfo(std::vector<const DegreeOfFreedom*>& rankDOFs,
				    std::map<const DegreeOfFreedom*, int>& boundaryDOFs);
129
130
131
132

    /// Removes all macro elements from the mesh that are not part of ranks partition.
    void removeMacroElements();

133
134
135
136
137
138
    void createLocalGlobalNumbering(std::vector<const DegreeOfFreedom*>& rankDOFs,
				    std::map<const DegreeOfFreedom*, int>& boundaryDOFs,
				    int& nRankDOFs, 
				    int& nOverallDOFs);

    void updateLocalGlobalNumbering(int& nRankDOFs, int& nOverallDOFs);
139

140
141
142
143
144
145
146
147
148
149
    /** \brief
     * This function traverses the whole mesh, i.e. before it is really partitioned,
     * and collects information about which DOF corresponds to which rank.
     *
     * @param[out] partionDOFs   Stores to each DOF pointer the set of ranks the DOF is
     *                           part of.
     * @param[out] rankDOFs      Stores all rank DOFs.
     * @param[out] boundaryDOFs  Stores all DOFs in ranks partition that are on an 
     *                           interior boundary but correspond to another rank.
     */
150
    void createDOFMemberInfo(std::map<const DegreeOfFreedom*, std::set<int> >& partitionDOFs,
151
152
			     std::vector<const DegreeOfFreedom*>& rankDOFs,
			     std::map<const DegreeOfFreedom*, int>& boundaryDOFs);
153
     
154

155
  protected:
156
157
158
159
160
161
    ///
    ProblemIterationInterface *iterationIF;

    ///
    ProblemTimeInterface *timeIF;

162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
    /// The rank of the current process.
    int mpiRank;

    /// Overall number of processes.
    int mpiSize;

    /** \brief
     * MPI communicator collected all processes, which should
     * be used for calculation. The Debug procces is not included
     * in this communicator.
     */
    MPI::Intracomm mpiComm;

    /// Name of the problem (as used in the init files)
    std::string name;

178
179
180
    /// Finite element space of the problem.
    FiniteElemSpace *feSpace;

181
182
183
    /// Mesh of the problem.
    Mesh *mesh;

184
185
186
    /// Refinement manager for the mesh.
    RefinementManager *refinementManager;

187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
    /// Pointer to the paritioner which is used to devide a mesh into partitions.
    ParMetisPartitioner *partitioner;

    /// Weights for the elements, i.e., the number of leaf elements within this element.
    std::map<int, double> elemWeights;

    /// Is true, if the mesh was not partitioned before, otherwise it's false.
    bool initialPartitionMesh;

    /** \brief
     * Stores to every coarse element index the number of the partition it 
     * corresponds to.
     */
    std::map<int, int> partitionVec;

    /** \brief
     * Stores an old partitioning of elements. To every element index the number
     * of the parition it corresponds to is stored.
     */
    std::map<int, int> oldPartitionVec;    
207

208
209
210
211
    Mat petscMatrix;

    Vec petscRhsVec;
    
212
    Vec petscSolVec;
213

214
215
    /// Number of DOFs in the rank mesh.
    int nRankDOFs;
Thomas Witkowski's avatar
Thomas Witkowski committed
216
217
218
219
220
221
222

    /** \brief 
     * Defines the interioir boundaries of the domain that result from partitioning
     * the whole mesh.
     */
    InteriorBoundary interiorBoundary;

223
224
225
226
227
228
229
230
231
232
233
    /** \brief
     * This map contains for each rank the list of dofs the current rank must send
     * to exchange solution dofs at the interior boundaries.
     */
    std::map<int, std::vector<DegreeOfFreedom> > sendDofs;

    /** \brief
     * This map contains for each rank the list of dofs from which the current rank 
     * must receive solution values of dofs at the interior boundaries.
     */
    std::map<int, std::vector<DegreeOfFreedom> > recvDofs;
234
235
236
237
238
239
240
241
242
243
244
245
246

    /// Maps local to global dof indices.
    std::map<DegreeOfFreedom, DegreeOfFreedom> mapLocalGlobalDOFs;

    /// Maps global to local dof indices.
    std::map<DegreeOfFreedom, DegreeOfFreedom> mapGlobalLocalDOFs;

    /** \brief
     * Maps all DOFs in ranks partition to a bool value. If it is true, the DOF is
     * owned by the rank. Otherwise, its an interior boundary DOF that is owned by
     * another rank.
     */
    std::map<DegreeOfFreedom, bool> isRankDOF;
247
248
249
250
251
252
253
254
255
  };

  class ParallelDomainProblemScal : public ParallelDomainProblemBase
  {
  public:
    ParallelDomainProblemScal(const std::string& name,
			      ProblemScal *problem,
			      ProblemInstatScal *problemInstat);

256
257
    void initParallelization(AdaptInfo *adaptInfo);

258
259
260
261
262
    virtual Flag oneIteration(AdaptInfo *adaptInfo, Flag toDo = FULL_ITERATION);

  protected:
    ProblemScal *probScal;

263
264
265
266
  };
}

#endif // AMDIS_PARALLELDOMAINPROBLEM_H