From 27235c554c9b3a42a04666bf015d071bd19eb961 Mon Sep 17 00:00:00 2001 From: Thomas Witkowski Date: Fri, 20 Apr 2012 05:30:26 +0000 Subject: [PATCH] Parallel type changes. --- AMDiS/src/AMDiS_fwd.h | 4 + AMDiS/src/DOFMatrix.cc | 16 +- AMDiS/src/DOFMatrix.h | 148 ++++++------- AMDiS/src/DOFVector.h | 205 ++++++++---------- AMDiS/src/DOFVector.hh | 12 +- AMDiS/src/parallel/MeshDistributor.cc | 67 +----- AMDiS/src/parallel/MeshDistributor.h | 32 +-- AMDiS/src/parallel/ParallelDebug.cc | 41 +--- AMDiS/src/parallel/ParallelDofMapping.h | 13 ++ AMDiS/src/parallel/ParallelTypes.h | 11 +- AMDiS/src/parallel/PeriodicMap.cc | 14 +- AMDiS/src/parallel/PeriodicMap.h | 9 - .../parallel/PetscSolverGlobalBlockMatrix.cc | 5 +- AMDiS/src/parallel/PetscSolverGlobalMatrix.cc | 8 +- AMDiS/src/parallel/PetscSolverSchur.cc | 4 + 15 files changed, 249 insertions(+), 340 deletions(-) diff --git a/AMDiS/src/AMDiS_fwd.h b/AMDiS/src/AMDiS_fwd.h index e3b96b80..c642788e 100644 --- a/AMDiS/src/AMDiS_fwd.h +++ b/AMDiS/src/AMDiS_fwd.h @@ -97,6 +97,10 @@ namespace AMDiS { class VertexInfo; class VertexVector; +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS + class FeSpaceDofMap; +#endif + struct BoundaryObject; struct AtomicBoundary; diff --git a/AMDiS/src/DOFMatrix.cc b/AMDiS/src/DOFMatrix.cc index 96bc1167..821174cc 100644 --- a/AMDiS/src/DOFMatrix.cc +++ b/AMDiS/src/DOFMatrix.cc @@ -26,6 +26,9 @@ #include "BoundaryManager.h" #include "Assembler.h" #include "Serializer.h" +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS +#include "parallel/ParallelDofMapping.h" +#endif namespace AMDiS { @@ -233,11 +236,12 @@ namespace AMDiS { if (condition && condition->isDirichlet()) { if (condition->applyBoundaryCondition()) { + #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - if ((*rankDofs)[rowIndices[i]]) - applyDBCs.insert(static_cast(row)); + if (dofMap->isRankDof(rowIndices[i])) + applyDBCs.insert(static_cast(row)); #else - applyDBCs.insert(static_cast(row)); + applyDBCs.insert(static_cast(row)); #endif } } else { @@ -514,4 +518,10 @@ namespace AMDiS { inserter = new inserter_type(matrix, nnz_per_row); } + + void DOFMatrix::setDofMap(FeSpaceDofMap& m) + { + dofMap = &m; + } + } diff --git a/AMDiS/src/DOFMatrix.h b/AMDiS/src/DOFMatrix.h index 9df5513e..2c3095ce 100644 --- a/AMDiS/src/DOFMatrix.h +++ b/AMDiS/src/DOFMatrix.h @@ -39,6 +39,8 @@ namespace AMDiS { + using namespace std; + /** \ingroup DOFAdministration * \brief * A DOFMatrix is a sparse matrix representation for matrices that work @@ -65,18 +67,17 @@ namespace AMDiS { /// Symbolic constant for an unused matrix entry static const int UNUSED_ENTRY = -1; - /** \brief - * Symbolic constant for an unused entry which is not followed by any - * used entry in this row - */ + /// Symbolic constant for an unused entry which is not followed by any + /// used entry in this row static const int NO_MORE_ENTRIES = -2; public: DOFMatrix(); /// Constructs a DOFMatrix with name n and the given row and olumn FeSpaces. - DOFMatrix(const FiniteElemSpace* rowFeSpace, const FiniteElemSpace* colFeSpace, - std::string n = ""); + DOFMatrix(const FiniteElemSpace* rowFeSpace, + const FiniteElemSpace* colFeSpace, + string n = ""); /// Copy-Constructor DOFMatrix(const DOFMatrix& rhs); @@ -102,15 +103,15 @@ namespace AMDiS { } // Only to get rid of the abstract functions, I hope they are not used - std::vector::iterator begin() + vector::iterator begin() { - ERROR_EXIT("Shouldn't be used, only fake."); std::vector v; + ERROR_EXIT("Shouldn't be used, only fake."); vector v; return v.begin(); } - std::vector::iterator end() + vector::iterator end() { - ERROR_EXIT("Shouldn't be used, only fake."); std::vector v; + ERROR_EXIT("Shouldn't be used, only fake."); vector v; return v.end(); } @@ -128,7 +129,7 @@ namespace AMDiS { } /// DOFMatrix does not need to be compressed before assembling, when using MTL4. - void compressDOFIndexed(int first, int last, std::vector &newDOF) + void compressDOFIndexed(int first, int last, vector &newDOF) {} /// Implements DOFIndexedBase::freeDOFContent() @@ -152,39 +153,38 @@ namespace AMDiS { /// Multiplication with a scalar. void scal(double s); - /** \brief - * Adds an operator to the DOFMatrix. A factor, that is multipled - * to the operator, and a multilier factor for the estimator may be - * also given. - */ - void addOperator(Operator *op, double* factor = NULL, double* estFactor = NULL); + /// Adds an operator to the DOFMatrix. A factor, that is multipled to the + /// operator, and a multilier factor for the estimator may be also given. + void addOperator(Operator *op, + double* factor = NULL, + double* estFactor = NULL); - inline std::vector::iterator getOperatorFactorBegin() + inline vector::iterator getOperatorFactorBegin() { return operatorFactor.begin(); } - inline std::vector::iterator getOperatorFactorEnd() + inline vector::iterator getOperatorFactorEnd() { return operatorFactor.end(); } - inline std::vector::iterator getOperatorEstFactorBegin() + inline vector::iterator getOperatorEstFactorBegin() { return operatorEstFactor.begin(); } - inline std::vector::iterator getOperatorEstFactorEnd() + inline vector::iterator getOperatorEstFactorEnd() { return operatorEstFactor.end(); } - inline std::vector::iterator getOperatorsBegin() + inline vector::iterator getOperatorsBegin() { return operators.begin(); } - inline std::vector::iterator getOperatorsEnd() + inline vector::iterator getOperatorsEnd() { return operators.end(); } @@ -237,11 +237,9 @@ namespace AMDiS { ElInfo* rowElInfo, ElInfo* colElInfo); - /* \brief - * That function must be called after the matrix assembling has been finished. - * This makes it possible to start some cleanup or matrix data compressing - * procedures. - */ + /// That function must be called after the matrix assembling has been + /// finished. This makes it possible to start some cleanup or matrix + /// data compressing procedures. void finishAssembling(); /** \brief @@ -252,10 +250,8 @@ namespace AMDiS { */ void startInsertion(int nnz_per_row = 10); - /** \brief - * Finishes insertion. For compressed matrix types, this is where the - * compression happens. - */ + /// Finishes insertion. For compressed matrix types, this is where the + /// compression happens. void finishInsertion() { FUNCNAME("DOFMatrix::finishInsertion()"); @@ -266,10 +262,8 @@ namespace AMDiS { inserter= 0; } - /** \brief - * Returns whether restriction should be performed after coarsening - * (false by default) - */ + /// Returns whether restriction should be performed after coarsening + /// (false by default) virtual bool coarseRestrict() { return false; @@ -299,17 +293,15 @@ namespace AMDiS { return num_rows(matrix); } - /** \brief - * Returns the number of used rows (equal to number of used DOFs in - * the row FE space). - */ + /// Returns the number of used rows (equal to number of used DOFs in + /// the row FE space). inline int getUsedSize() const { return rowFeSpace->getAdmin()->getUsedSize(); } /// Returns \ref name - inline std::string getName() const + inline string getName() const { return name; } @@ -326,10 +318,8 @@ namespace AMDiS { /// Changes col at logical indices a,b to c void changeColOfEntry(DegreeOfFreedom a, DegreeOfFreedom b, DegreeOfFreedom c); - /** \brief - * Creates an entry with logical indices irow, icol if there is no entry - * yet. Than sign * entry is added to the value at this logical indices - */ + /// Creates an entry with logical indices irow, icol if there is no entry + /// yet. Than sign * entry is added to the value at this logical indices void addSparseDOFEntry(double sign, int irow, int jcol, double entry, bool add = true); @@ -350,17 +340,17 @@ namespace AMDiS { bool symmetric(); - inline std::vector& getOperators() + inline vector& getOperators() { return operators; } - inline std::vector& getOperatorFactor() + inline vector& getOperatorFactor() { return operatorFactor; } - inline std::vector& getOperatorEstFactor() + inline vector& getOperatorEstFactor() { return operatorEstFactor; } @@ -399,65 +389,53 @@ namespace AMDiS { } /// Writes the matrix to an output stream. - void serialize(std::ostream &out); + void serialize(ostream &out); /// Reads a matrix from an input stream. - void deserialize(std::istream &in); + void deserialize(istream &in); /// int memsize(); #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - void setRankDofs(std::map& dofmap) - { - rankDofs = &dofmap; - } + void setDofMap(FeSpaceDofMap& m); #endif protected: - /** \brief - * Pointer to a FiniteElemSpace with information about corresponding row DOFs - * and basis functions - */ + /// Pointer to a FiniteElemSpace with information about corresponding row DOFs + /// and basis functions const FiniteElemSpace *rowFeSpace; - /** \brief - * Pointer to a FiniteElemSpace with information about corresponding - * column DOFs and basis functions - */ + /// Pointer to a FiniteElemSpace with information about corresponding + /// column DOFs and basis functions const FiniteElemSpace *colFeSpace; /// Name of the DOFMatrix - std::string name; + string name; - /// Sparse matrix, type is a template parameter by default compressed2D + /// Sparse matrix, type is a template parameter by + /// default compressed2D base_matrix_type matrix; /// Used while mesh traversal static DOFMatrix *traversePtr; - /** \brief - * Pointers to all operators of the equation systems. Are used in the - * assembling process. - */ - std::vector operators; + /// Pointers to all operators of the equation systems. Are used in the + /// assembling process. + vector operators; - /** \brief - * Defines for each operator a factor which is used to scal the element - * matrix after the assembling process of the operator. - */ - std::vector operatorFactor; + /// Defines for each operator a factor which is used to scal the element + /// matrix after the assembling process of the operator. + vector operatorFactor; /// - std::vector operatorEstFactor; + vector operatorEstFactor; /// BoundaryManager *boundaryManager; - /** \brief - * If false, the matrix is a diagonal matrix within a matrix of DOF matrices. - * Otherwise the value is true, and the matrix is an off-diagonal matrix. - */ + /// If false, the matrix is a diagonal matrix within a matrix of DOF matrices. + /// Otherwise the value is true, and the matrix is an off-diagonal matrix. bool coupleMatrix; /// Temporary variable used in assemble() @@ -470,10 +448,10 @@ namespace AMDiS { int nCol; /// Maps local row indices of an element to global matrix indices. - std::vector rowIndices; + vector rowIndices; /// Maps local col indices of an element to global matrix indices. - std::vector colIndices; + vector colIndices; /* \brief * A set of row indices. When assembling the DOF matrix, all rows, that @@ -492,10 +470,10 @@ namespace AMDiS { int nnzPerRow; #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - /// Stores for the DOFs of the row FE spaces whether they are owned by the - /// rank or not. This is used to ensure that Dirichlet BC is handled - /// correctly in parallel computations. - std::map *rankDofs; + /// Is used in parallel computations to check whether specific DOFs in the + /// row FE spaces are owned by the rank or not. This is used to ensure that + /// Dirichlet BC is handled correctly in parallel computations. + FeSpaceDofMap *dofMap; #endif /// Inserter object: implemented as pointer, allocated and deallocated as needed diff --git a/AMDiS/src/DOFVector.h b/AMDiS/src/DOFVector.h index 893ded7e..94813642 100644 --- a/AMDiS/src/DOFVector.h +++ b/AMDiS/src/DOFVector.h @@ -42,9 +42,14 @@ #include "BasisFunction.h" #include "FiniteElemSpace.h" #include "SurfaceQuadrature.h" +#ifdef HAVE_PARALLEL_DOMAIN_AMDIS +#include "parallel/ParallelDofMapping.h" +#endif namespace AMDiS { + using namespace std; + template class DOFVectorBase : public DOFIndexed { @@ -57,25 +62,21 @@ namespace AMDiS { nBasFcts(0) { #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - rankDofs = NULL; + dofMap = NULL; #endif } - DOFVectorBase(const FiniteElemSpace *f, std::string n); + DOFVectorBase(const FiniteElemSpace *f, string n); virtual ~DOFVectorBase(); - /** \brief - * For the given element, this function returns an array of all DOFs of this - * DOFVector that are defined on this element. - */ + /// For the given element, this function returns an array of all DOFs of + /// this DOFVector that are defined on this element. virtual void getLocalVector(const Element *el, mtl::dense_vector& localVec) const; - /** \brief - * Evaluates the DOF vector at a set of quadrature points defined on the - * given element. - */ + /// Evaluates the DOF vector at a set of quadrature points defined on the + /// given element. void getVecAtQPs(const ElInfo *elInfo, const Quadrature *quad, const FastQuadrature *quadFast, @@ -87,10 +88,8 @@ namespace AMDiS { const FastQuadrature *quadFast, mtl::dense_vector& vecAtQPs) const; - /** \brief - * Evaluates the gradient of a DOF vector at a set of quadrature points defined on the - * given element. - */ + /// Evaluates the gradient of a DOF vector at a set of quadrature points + /// defined on the given element. void getGrdAtQPs( const ElInfo *elInfo, const Quadrature *quad, const FastQuadrature *quadFast, @@ -102,10 +101,8 @@ namespace AMDiS { const FastQuadrature *quadFast, mtl::dense_vector::type> &grdAtQPs) const; - /** \brief - * Evaluates the comp'th component of the derivative of a DOF vector at a set of quadrature points defined on the - * given element. - */ + /// Evaluates the comp'th component of the derivative of a DOF vector at a + /// set of quadrature points defined on the given element. void getDerivativeAtQPs( const ElInfo *elInfo, const Quadrature *quad, const FastQuadrature *quadFast, @@ -119,10 +116,8 @@ namespace AMDiS { int comp, mtl::dense_vector &derivativeAtQPs) const; - /** \brief - * Evaluates the jacobian of a DOF vector at a set of quadrature points defined on the - * given element. - */ + /// Evaluates the jacobian of a DOF vector at a set of quadrature points + /// defined on the given element. void getD2AtQPs(const ElInfo *elInfo, const Quadrature *quad, const FastQuadrature *quadFast, @@ -134,10 +129,8 @@ namespace AMDiS { return feSpace; } - /** \brief - * Assembles the element vector for the given ellement and adds the - * element matrix to the current DOF vector. - */ + /// Assembles the element vector for the given ellement and adds the + /// element matrix to the current DOF vector. void assemble(T factor, ElInfo *elInfo, const BoundaryType *bound, Operator *op = NULL); @@ -154,11 +147,9 @@ namespace AMDiS { ElInfo *elInfo, bool add = true); - /* \brief - * That function must be called after the matrix assembling has been finished. - * This makes it possible to start some cleanup or matrix data compressing - * procedures. - */ + /// That function must be called after the matrix assembling has been + /// finished. This makes it possible to start some cleanup or matrix + /// data compressing procedures. void finishAssembling(); inline void addOperator(Operator* op, @@ -170,33 +161,33 @@ namespace AMDiS { operatorEstFactor.push_back(estFactor); } - inline std::vector::iterator getOperatorFactorBegin() + inline vector::iterator getOperatorFactorBegin() { return operatorFactor.begin(); } - inline std::vector::iterator getOperatorFactorEnd() + inline vector::iterator getOperatorFactorEnd() { return operatorFactor.end(); } - inline std::vector::iterator getOperatorEstFactorBegin() + inline vector::iterator getOperatorEstFactorBegin() { return operatorEstFactor.begin(); } - inline std::vector::iterator getOperatorEstFactorEnd() + inline vector::iterator getOperatorEstFactorEnd() { return operatorEstFactor.end(); } - inline std::vector::iterator getOperatorsBegin() + inline vector::iterator getOperatorsBegin() { return operators.begin(); } - inline std::vector::iterator getOperatorsEnd() + inline vector::iterator getOperatorsEnd() { return operators.end(); } @@ -211,28 +202,28 @@ namespace AMDiS { */ T evalUh(const DimVec& lambda, DegreeOfFreedom* ind); - inline std::vector& getOperators() + inline vector& getOperators() { return operators; } - inline std::vector& getOperatorFactor() + inline vector& getOperatorFactor() { return operatorFactor; } - inline std::vector& getOperatorEstFactor() + inline vector& getOperatorEstFactor() { return operatorEstFactor; } /// Returns \ref name - inline std::string getName() const + inline string getName() const { return name; } - inline void setName(std::string n) + inline void setName(string n) { name = n; } @@ -248,16 +239,16 @@ namespace AMDiS { } #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - inline void setRankDofs(std::map &dofmap) + void setDofMap(FeSpaceDofMap& m) { - rankDofs = &dofmap; + dofMap = &m; } inline bool isRankDof(DegreeOfFreedom dof) { - TEST_EXIT_DBG(rankDofs)("No rank dofs set!\n"); + TEST_EXIT_DBG(dofMap)("No rank dofs set!\n"); - return (*rankDofs)[dof]; + return dofMap->isRankDof(dof); } #endif @@ -266,19 +257,19 @@ namespace AMDiS { const FiniteElemSpace *feSpace; /// - std::string name; + string name; /// ElementVector elementVector; /// - std::vector operators; + vector operators; /// - std::vector operatorFactor; + vector operatorFactor; /// - std::vector operatorEstFactor; + vector operatorEstFactor; /// BoundaryManager *boundaryManager; @@ -290,7 +281,7 @@ namespace AMDiS { int dim; #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - std::map *rankDofs; + FeSpaceDofMap *dofMap; #endif }; @@ -362,10 +353,10 @@ namespace AMDiS { {} /// Constructs a DOFVector with name n belonging to FiniteElemSpace f - DOFVector(const FiniteElemSpace* f, std::string n); + DOFVector(const FiniteElemSpace* f, string n); /// Initialization. - void init(const FiniteElemSpace* f, std::string n); + void init(const FiniteElemSpace* f, string n); /// Copy Constructor DOFVector(const DOFVector& rhs) @@ -380,23 +371,21 @@ namespace AMDiS { virtual ~DOFVector(); /// Returns iterator to the begin of \ref vec - typename std::vector::iterator begin() + typename vector::iterator begin() { return vec.begin(); } /// Returns iterator to the end of \ref vec - typename std::vector::iterator end() + typename vector::iterator end() { return vec.end(); } - /** \brief - * Used by DOFAdmin to compress this DOFVector. Implementation of - * DOFIndexedBase::compress() - */ + /// Used by DOFAdmin to compress this DOFVector. Implementation of + /// \ref DOFIndexedBase::compress() virtual void compressDOFIndexed(int first, int last, - std::vector &newDof); + vector &newDof); /// Sets \ref coarsenOperation to op inline void setCoarsenOperation(CoarsenOperation op) @@ -417,7 +406,7 @@ namespace AMDiS { inline void refineInterpol(RCNeighbourList&, int) {} /// Returns \ref vec - std::vector& getVector() + vector& getVector() { return vec; } @@ -489,10 +478,8 @@ namespace AMDiS { double Int(int meshLevel, Quadrature* q = NULL) const; - /** \brief - * Calculates Integral of this DOFVector over parts of the domain - * boundary, indicated by boundaryType. Implemented for DOFVector - **/ + /// Calculates Integral of this DOFVector over parts of the domain + /// boundary, indicated by boundaryType. Implemented for DOFVector T IntOnBoundary(BoundaryType boundary, Quadrature* q = NULL) const { FUNCNAME("DOFVector::IntOnBoundary())"); @@ -500,11 +487,9 @@ namespace AMDiS { return 0.0; } - /** \brief - * Calculates Integral of this DOFVector times normal vector over parts - * of the domain boundary, indicated by boundaryType. Implemented for - * DOFVector > - **/ + /// Calculates Integral of this DOFVector times normal vector over parts + /// of the domain boundary, indicated by boundaryType. Implemented for + /// DOFVector > double IntOnBoundaryNormal(BoundaryType boundary, Quadrature* q = NULL) const { FUNCNAME("DOFVector::IntOnBoundaryNormal())"); @@ -594,34 +579,35 @@ namespace AMDiS { /// int calcMemoryUsage() const; - /** \brief - * Computes the coefficients of the interpolant of the function fct and - * stores these in the DOFVector - */ + /// Computes the coefficients of the interpolant of the function fct and + /// stores these in the DOFVector void interpol(AbstractFunction > *fct); void interpol(DOFVector *v, double factor = 1.0); - /** eval DOFVector at given point p. If oldElInfo != NULL the search for the element, where p is inside, - * starts from oldElInfo. - * implemented for: double, WorldVector< double > - */ - inline const T& evalAtPoint( - WorldVector &p, ElInfo *oldElInfo = NULL, T* value = NULL) const + /// Eval DOFVector at given point p. If oldElInfo != NULL the search for + /// the element, where p is inside, starts from oldElInfo. implemented for: + /// double, WorldVector< double > + inline const T& evalAtPoint(WorldVector &p, + ElInfo *oldElInfo = NULL, + T* value = NULL) const { FUNCNAME("DOFVector::evalAtPoint())"); TEST_EXIT(false)("Please implement your evaluation\n"); return *value; } - /** determine the DegreeOfFreedom that has coords with minimal euclidean distance to WorldVector p - * return true if DOF is found, and false otherwise - */ - const bool getDOFidxAtPoint(WorldVector &p, DegreeOfFreedom &idx, ElInfo *oldElInfo = NULL, bool useOldElInfo = false) const; + /// Determine the DegreeOfFreedom that has coords with minimal euclidean + /// distance to WorldVector p. return true if DOF is found, and false + /// otherwise. + const bool getDofIdxAtPoint(WorldVector &p, + DegreeOfFreedom &idx, + ElInfo *oldElInfo = NULL, + bool useOldElInfo = false) const; /// Writes the data of the DOFVector to an output stream. - void serialize(std::ostream &out) + void serialize(ostream &out) { unsigned int size = vec.size(); out.write(reinterpret_cast(&size), sizeof(unsigned int)); @@ -629,7 +615,7 @@ namespace AMDiS { } /// Reads data of an DOFVector from an input stream. - void deserialize(std::istream &in) + void deserialize(istream &in) { unsigned int size; in.read(reinterpret_cast(&size), sizeof(unsigned int)); @@ -637,20 +623,18 @@ namespace AMDiS { in.read(reinterpret_cast(&(vec[0])), size * sizeof(T)); } -// DOFVector > *getGradient(DOFVector >*) const; DOFVector::type>* getGradient(DOFVector::type> *grad) const; WorldVector*> *getGradient(WorldVector*> *grad) const; -// DOFVector >* getRecoveryGradient(DOFVector >*) const; DOFVector::type>* getRecoveryGradient(DOFVector::type> *grad) const; protected: /// Data container - std::vector vec; + vector vec; /// Specifies what operation should be performed after coarsening CoarsenOperation coarsenOperation; @@ -698,11 +682,9 @@ namespace AMDiS { public DOFContainer { public: - /** \brief - * Calls constructor of DOFVector and registers itself - * as DOFContainer at DOFAdmin - */ - DOFVectorDOF(const FiniteElemSpace* feSpace_, std::string name_) + /// Calls constructor of DOFVector and registers itself + /// as DOFContainer at DOFAdmin + DOFVectorDOF(const FiniteElemSpace* feSpace_, string name_) : DOFVector(feSpace_, name_) { feSpace->getAdmin()->addDOFContainer(this); @@ -714,10 +696,8 @@ namespace AMDiS { feSpace->getAdmin()->removeDOFContainer(this); } - /** \brief - * Implements DOFContainer::operator[]() by calling - * DOFVector::operator[]() - */ + /// Implements DOFContainer::operator[]() by calling + /// DOFVector::operator[]() DegreeOfFreedom& operator[](DegreeOfFreedom i) { return DOFVector::operator[](i); @@ -870,7 +850,7 @@ namespace AMDiS { } template - inline void checkFeSpace(const FiniteElemSpace* feSpace, const std::vector& vec) + inline void checkFeSpace(const FiniteElemSpace* feSpace, const vector& vec) { TEST_EXIT_DBG(feSpace)("feSpace is NULL\n"); TEST_EXIT_DBG(feSpace->getAdmin())("admin is NULL\n"); @@ -883,8 +863,8 @@ namespace AMDiS { WorldVector*> *result); template - std::vector*> *transform(DOFVector::type> *vec, - std::vector*> *res); + vector*> *transform(DOFVector::type> *vec, + vector*> *res); /// Computes the integral: \f$ \int f(\vec{x})\,\text{d}\vec{x}\f$ @@ -914,8 +894,8 @@ namespace AMDiS { /** \brief * Computes the integral of a function that includes two different DOFVectors: * \f$ \int f(v(\vec{x}), w(\vec{x}))\,\text{d}\vec{x}\f$ - * This function works also for the case that the DOFVectors are defined on two different - * meshes. + * This function works also for the case that the DOFVectors are defined on + * two different meshes. */ template TOut integrate_Vec2(const DOFVector &vec1, @@ -930,13 +910,15 @@ namespace AMDiS { return integrate_Vec2(vec1, vec2, fct); } - /// Computes the integral of a function with two DOFVectors defined on the same mesh. + /// Computes the integral of a function with two DOFVectors defined on the + /// same mesh. template TOut int_Vec2_SingleMesh(const DOFVector &vec1, const DOFVector &vec2, BinaryAbstractFunction *fct); - /// Computes the integral of a function with two DOFVectors defined on different meshes. + /// Computes the integral of a function with two DOFVectors defined on + /// different meshes. template TOut int_Vec2_DualMesh(const DOFVector &vec1, const DOFVector &vec2, @@ -960,13 +942,14 @@ namespace AMDiS { } /// Computes the integral: \f$ \int f(\{v_i(\vec{x})\}_i)\,\text{d}\vec{x}\f$ - double integrateGeneral(const std::vector*> &vecs, - AbstractFunction > *fct); + double integrateGeneral(const vector*> &vecs, + AbstractFunction > *fct); - /// Computes the integral: \f$ \int f(\{v_i(\vec{x})\}_i,\{\nabla w_i(\vec{x})\}_i)\,\text{d}\vec{x}\f$ - double integrateGeneralGradient(const std::vector*> &vecs, - const std::vector*> &grds, - BinaryAbstractFunction, std::vector > > *fct); + /// Computes the integral: + /// \f$ \int f(\{v_i(\vec{x})\}_i,\{\nabla w_i(\vec{x})\}_i)\,\text{d}\vec{x}\f$ + double integrateGeneralGradient(const vector*> &vecs, + const vector*> &grds, + BinaryAbstractFunction, vector > > *fct); } diff --git a/AMDiS/src/DOFVector.hh b/AMDiS/src/DOFVector.hh index 30b165f7..b4bb8152 100644 --- a/AMDiS/src/DOFVector.hh +++ b/AMDiS/src/DOFVector.hh @@ -946,8 +946,12 @@ namespace AMDiS { template - const bool DOFVector::getDOFidxAtPoint(WorldVector &p, DegreeOfFreedom &idx, ElInfo *oldElInfo, bool useOldElInfo) const - { FUNCNAME("DOFVector::getDOFidxAtPoint()"); + const bool DOFVector::getDofIdxAtPoint(WorldVector &p, + DegreeOfFreedom &idx, + ElInfo *oldElInfo, + bool useOldElInfo) const + { + FUNCNAME("DOFVector::getDofIdxAtPoint()"); Mesh *mesh = this->feSpace->getMesh(); const BasisFunction *basFcts = this->feSpace->getBasisFcts(); @@ -994,7 +998,7 @@ namespace AMDiS { if(!oldElInfo) delete elInfo; return inside; - }; + } template @@ -1052,7 +1056,7 @@ namespace AMDiS { } #ifdef HAVE_PARALLEL_DOMAIN_AMDIS - this->rankDofs = rhs.rankDofs; + this->dofMap = rhs.dofMap; #endif return *this; diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc index b4cd1ff9..d54cd0e7 100644 --- a/AMDiS/src/parallel/MeshDistributor.cc +++ b/AMDiS/src/parallel/MeshDistributor.cc @@ -695,7 +695,7 @@ namespace AMDiS { TEST_EXIT(find(feSpaces.begin(), feSpaces.end(), rowFeSpace) != feSpaces.end()) ("Should not happen!\n"); - probStat->getSystemMatrix(i, j)->setRankDofs(dofFeData[rowFeSpace].isRankDof); + probStat->getSystemMatrix(i, j)->setDofMap(dofMap[rowFeSpace]); } @@ -710,8 +710,8 @@ namespace AMDiS { TEST_EXIT(find(feSpaces.begin(), feSpaces.end(), feSpace) != feSpaces.end()) ("Should not happen!\n"); - probStat->getRhsVector(i)->setRankDofs(dofFeData[feSpace].isRankDof); - probStat->getSolution(i)->setRankDofs(dofFeData[feSpace].isRankDof); + probStat->getRhsVector(i)->setDofMap(dofMap[feSpace]); + probStat->getSolution(i)->setDofMap(dofMap[feSpace]); } } @@ -2015,8 +2015,6 @@ namespace AMDiS { DofContainer rankDofs(rankDofSet.begin(), rankDofSet.end()); sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue); - int nRankAllDofs = rankDofs.size(); - // === Traverse interior boundaries and get all DOFs on them. === @@ -2042,27 +2040,15 @@ namespace AMDiS { for (; !it.endDofIter(); it.nextDof()) dofMap[feSpace].insert(it.getDofIndex()); - // === Send and receive new DOF indices. === + // === Update DOF admins due to new number of DOFs. === + + lastMeshChangeIndex = mesh->getChangeIndex(); #if (DEBUG != 0) ParallelDebug::testDofContainerCommunication(*this, sendDofs.getData(), recvDofs.getData()); #endif - - // First, we set all DOFs in ranks partition to be owend by the rank. Than, - // the DOFs in ranks partition that are owned by other rank are set to false. - dofFeData[feSpace].isRankDof.clear(); - for (int i = 0; i < nRankAllDofs; i++) - dofFeData[feSpace].isRankDof[i] = true; - - for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank()) - for (; !it.endDofIter(); it.nextDof()) - dofFeData[feSpace].isRankDof[it.getDofIndex()] = false; - - // === Update DOF admins due to new number of DOFs. === - - lastMeshChangeIndex = mesh->getChangeIndex(); } @@ -2278,15 +2264,11 @@ namespace AMDiS { { FUNCNAME("MeshDistributor::mapGlobalToLocal()"); - TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n"); - - ERROR_EXIT("REIMPLEMENT THAT SHIT!\n"); - /* - for (DofMap::iterator it = dofFeData[feSpace].mapDofToGlobal.begin(); - it != dofFeData[feSpace].mapDofToGlobal.end(); ++it) - if (it->second == dof) + for (map::iterator it = dofMap[feSpace].getMap().begin(); + it != dofMap[feSpace].getMap().end(); ++it) + if (it->second.global == dof) return it->first; - */ + return -1; } @@ -2313,18 +2295,7 @@ namespace AMDiS { // === Serialieze FE space dependent data === - unsigned int nFeSpace = feSpaces.size(); - SerUtil::serialize(out, nFeSpace); - - for (unsigned int i = 0; i < nFeSpace; i++) { - // SerUtil::serialize(out, dofFeData[feSpaces[i]].nRankDofs); - // SerUtil::serialize(out, dofFeData[feSpaces[i]].nOverallDofs); - // SerUtil::serialize(out, dofFeData[feSpaces[i]].rStartDofs); - - SerUtil::serialize(out, dofFeData[feSpaces[i]].isRankDof); - // SerUtil::serialize(out, dofFeData[feSpaces[i]].mapDofToGlobal); - // SerUtil::serialize(out, dofFeData[feSpaces[i]].mapLocalToDof); - } + dofMap.serialize(out); periodicMap.serialize(out, feSpaces); @@ -2385,21 +2356,7 @@ namespace AMDiS { // === Deerialieze FE space dependent data === - unsigned int nFeSpace = 0; - SerUtil::deserialize(in, nFeSpace); - TEST_EXIT(nFeSpace == feSpaces.size()) - ("Number of FE spaces is %d, but %d are stored in serialization file!\n", - feSpaces.size(), nFeSpace); - - for (unsigned int i = 0; i < nFeSpace; i++) { -// SerUtil::deserialize(in, dofFeData[feSpaces[i]].nRankDofs); -// SerUtil::deserialize(in, dofFeData[feSpaces[i]].nOverallDofs); -// SerUtil::deserialize(in, dofFeData[feSpaces[i]].rStartDofs); - - SerUtil::deserialize(in, dofFeData[feSpaces[i]].isRankDof); - // SerUtil::deserialize(in, dofFeData[feSpaces[i]].mapDofToGlobal); - // SerUtil::deserialize(in, dofFeData[feSpaces[i]].mapLocalToDof); - } + dofMap.deserialize(in); periodicMap.deserialize(in, feSpaces); diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h index f1aef6c0..9bb6d721 100644 --- a/AMDiS/src/parallel/MeshDistributor.h +++ b/AMDiS/src/parallel/MeshDistributor.h @@ -55,15 +55,6 @@ namespace AMDiS { }; - struct DofData - { - /// Maps all DOFs in ranks partition to a bool value. If it is true, the DOF - /// is owned by the rank. Otherwise, its an interior boundary DOF that is - /// owned by another rank. - DofIndexToBool isRankDof; - }; - - class MeshDistributor { private: @@ -222,11 +213,18 @@ namespace AMDiS { /// Returns for a global index the DOF index in rank's subdomain. As there /// is no direct data structure that stores this information, we have to - /// search for it in \ref dofFeData.mapDofToGlobal. This is not very - /// efficient and this function should thus be used for debugging only. + /// search for it in \ref dofMap. This is not very efficient and this + /// function should thus be used for debugging only. DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace, DegreeOfFreedom dof); + /// Maps a local DOF to its local index. + inline DegreeOfFreedom mapLocalToDof(const FiniteElemSpace *feSpace, + DegreeOfFreedom dof) + { + return dofMap[feSpace][dof].local; + } + /// Returns the periodic mapping handler, \ref periodicMap. inline PeriodicMap& getPeriodicMap() { @@ -252,15 +250,7 @@ namespace AMDiS { /// is in rank's partition, but is owned by some other rank. inline bool getIsRankDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof) { - if (dofFeData[feSpace].isRankDof.count(dof)) - return dofFeData[feSpace].isRankDof[dof]; - - return false; - } - - inline DofIndexToBool& getIsRankDof(const FiniteElemSpace *feSpace) - { - return dofFeData[feSpace].isRankDof; + return dofMap[feSpace].isRankDof(dof); } inline long getLastMeshChangeIndex() @@ -565,8 +555,6 @@ namespace AMDiS { /// macro element. map partitionMap; - map dofFeData; - ParallelDofMapping dofMap; /// Database to store and query all sub-objects of all elements of the diff --git a/AMDiS/src/parallel/ParallelDebug.cc b/AMDiS/src/parallel/ParallelDebug.cc index ece764f4..4405af98 100644 --- a/AMDiS/src/parallel/ParallelDebug.cc +++ b/AMDiS/src/parallel/ParallelDebug.cc @@ -198,7 +198,7 @@ namespace AMDiS { for (PeriodicDofMap::iterator it = otherMap.begin(); it != otherMap.end(); ++it) { - for (DofMap::iterator dofIt = it->second.begin(); + for (std::map::iterator dofIt = it->second.begin(); dofIt != it->second.end(); ++dofIt) { if (dofMap.count(it->first) == 1 && dofMap[it->first].count(dofIt->first) == 1) { @@ -217,7 +217,7 @@ namespace AMDiS { for (PeriodicDofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) { - for (DofMap::iterator dofIt = it->second.begin(); + for (std::map::iterator dofIt = it->second.begin(); dofIt != it->second.end(); ++dofIt) { if (it->second[dofIt->second] != dofIt->first) { MSG("[DBG] For boundary type %d: DOF %d -> %d, but %d -> %d!\n", @@ -668,42 +668,7 @@ namespace AMDiS { { FUNCNAME("ParallelDebug::printMapPeriodic()"); - ERROR_EXIT("Function must be rewritten!\n"); - -#if 0 - PeriodicMap &perMap = pdb.getPeriodicMap(); - - const FiniteElemSpace* feSpace = pdb.feSpaces[0]; - - typedef map DofMap; - typedef map > PeriodicDofMap; - - if (rank == -1 || pdb.mpiRank == rank) { - cout << "====== DOF MAP PERIODIC ====== " << endl; - - for (PeriodicDofMap::iterator it = perMap.periodicDofMap.begin(); - it != perMap.periodicDofMap.end(); ++it) { - cout << "DOF MAP " << it->first << ": "; - for (std::set::iterator dofit = it->second.begin(); - dofit != it->second.end(); ++dofit) - cout << *dofit << " "; - cout << endl; - - DegreeOfFreedom localdof = -1; - map &dofMap = pdb.dofMap[feSpace].getMap(); - for (map::iterator it = dofMap.begin(); - it != dofMap.end(); ++it) - if (dofIt->second == it->first) - localdof = dofIt->first; - - TEST_EXIT(localdof != -1)("There is something wrong!\n"); - - WorldVector coords; - pdb.mesh->getDofIndexCoords(localdof, feSpace, coords); - coords.print(); - } - } -#endif + ERROR_EXIT("Function must be rewritten, check svn for old code!\n"); } diff --git a/AMDiS/src/parallel/ParallelDofMapping.h b/AMDiS/src/parallel/ParallelDofMapping.h index ce40a27a..63ebf27d 100644 --- a/AMDiS/src/parallel/ParallelDofMapping.h +++ b/AMDiS/src/parallel/ParallelDofMapping.h @@ -32,6 +32,7 @@ #define AMDIS_FE_SPACE_MAPPING_H namespace AMDiS { + using namespace std; /// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local @@ -366,6 +367,18 @@ namespace AMDiS { return dofToMatIndex.get(ithComponent, d) - rStartDofs; } + // Writes all data of this object to an output stream. + void serialize(ostream &out) + { + ERROR_EXIT("MUST BE IMPLEMENTED!\n"); + } + + // Reads the object data from an input stream. + void deserialize(istream &in) + { + ERROR_EXIT("MUST BE IMPLEMENTED!\n"); + } + protected: /// Insert a new FE space DOF mapping for a given FE space. void addFeSpace(const FiniteElemSpace* feSpace); diff --git a/AMDiS/src/parallel/ParallelTypes.h b/AMDiS/src/parallel/ParallelTypes.h index a0f3dba4..43edc233 100644 --- a/AMDiS/src/parallel/ParallelTypes.h +++ b/AMDiS/src/parallel/ParallelTypes.h @@ -49,7 +49,7 @@ namespace AMDiS { typedef map RankToDofContainer; /// Defines a mapping type from DOF indices to DOF indices. - typedef map DofMap; + //typedef map DofMap; /// Defines a mapping type from DOFs to boolean values. typedef map DofToBool; @@ -61,6 +61,15 @@ namespace AMDiS { typedef InteriorBoundary::RankToBoundMap RankToBoundMap; typedef map DofIndexMap; + + /// Maps a boundary type, i.e., a boundary identifier index, to a periodic + /// DOF mapping. + typedef std::map > PeriodicDofMap; + + /// Different FE spaces may have different DOFs on the same mesh. Thus we + /// need to have a periodic DOF mapping for each FE space. + typedef std::map PeriodicDofMapFeSpace; + typedef vector MeshCodeVec; } diff --git a/AMDiS/src/parallel/PeriodicMap.cc b/AMDiS/src/parallel/PeriodicMap.cc index 31636ce0..558a46ce 100644 --- a/AMDiS/src/parallel/PeriodicMap.cc +++ b/AMDiS/src/parallel/PeriodicMap.cc @@ -10,20 +10,26 @@ // See also license.opensource.txt in the distribution. +#include #include +#include "Global.h" #include "parallel/PeriodicMap.h" +#include "parallel/ParallelTypes.h" namespace AMDiS { + using namespace std; + void PeriodicMap::add(const FiniteElemSpace *feSpace, PeriodicDofMap &newMap) { FUNCNAME("PeriodicMap::add()"); - for (PeriodicDofMap::iterator it = newMap.begin(); it != newMap.end(); ++it) - for (DofMap::iterator dofIt =it->second.begin(); + for (PeriodicDofMap::iterator it = newMap.begin(); it != newMap.end(); ++it) { + for (std::map::iterator dofIt = it->second.begin(); dofIt != it->second.end(); ++dofIt) add(feSpace, it->first, dofIt->second, dofIt->first); + } } @@ -64,7 +70,7 @@ namespace AMDiS { for (PeriodicDofMap::iterator it = data.begin(); it != data.end(); ++it) { int type = it->first; - DofMap dofMap = it->second; + std::map dofMap = it->second; SerUtil::serialize(out, type); SerUtil::serialize(out, dofMap); @@ -97,7 +103,7 @@ namespace AMDiS { for (int i = 0; i < mapSize; i++) { int type; - DofMap dofMap; + std::map dofMap; SerUtil::deserialize(in, type); SerUtil::deserialize(in, dofMap); diff --git a/AMDiS/src/parallel/PeriodicMap.h b/AMDiS/src/parallel/PeriodicMap.h index 11da11da..8ed5e2e7 100644 --- a/AMDiS/src/parallel/PeriodicMap.h +++ b/AMDiS/src/parallel/PeriodicMap.h @@ -32,15 +32,6 @@ namespace AMDiS { - /// Maps a boundary type, i.e., a boundary identifier index, to a periodic - /// DOF mapping. - typedef std::map PeriodicDofMap; - - /// Different FE spaces may have different DOFs on the same mesh. Thus we - /// need to have a periodic DOF mapping for each FE space. - typedef std::map PeriodicDofMapFeSpace; - - /** \brief * This class stores information about the periodic DOFs in the (sub)domain. * To each DOF on a periodic boundary there is the information to which DOF diff --git a/AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.cc b/AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.cc index 410c420c..0567c2c0 100644 --- a/AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.cc +++ b/AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.cc @@ -154,9 +154,8 @@ namespace AMDiS { PetscScalar *vecPointer; VecGetArray(tmp, &vecPointer); - ERROR_EXIT("REWRITE CODE!\n"); - // for (int j = 0; j < nRankDofs; j++) - // dofvec[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[j]; + for (int j = 0; j < nRankDofs; j++) + dofvec[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[j]; VecRestoreArray(tmp, &vecPointer); } diff --git a/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc b/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc index 9a963d28..fb8c0ac5 100644 --- a/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc +++ b/AMDiS/src/parallel/PetscSolverGlobalMatrix.cc @@ -45,8 +45,7 @@ namespace AMDiS { int recvAllValues = 0; int sendValue = static_cast(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz); - meshDistributor->getMpiComm().Allreduce(&sendValue, &recvAllValues, - 1, MPI_INT, MPI_SUM); + mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM); recvAllValues = 1; @@ -197,9 +196,8 @@ namespace AMDiS { const FiniteElemSpace *feSpace = dv.getFeSpace(); int nRankDofs = meshDistributor->getNumberRankDofs(feSpace); - ERROR_EXIT("REWRITE CODE!\n"); - // for (int j = 0; j < nRankDofs; j++) - // dv[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[c++]; + for (int j = 0; j < nRankDofs; j++) + dv[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[c++]; } VecRestoreArray(petscSolVec, &vecPointer); diff --git a/AMDiS/src/parallel/PetscSolverSchur.cc b/AMDiS/src/parallel/PetscSolverSchur.cc index dd1a951d..a8c6e32d 100644 --- a/AMDiS/src/parallel/PetscSolverSchur.cc +++ b/AMDiS/src/parallel/PetscSolverSchur.cc @@ -100,6 +100,9 @@ namespace AMDiS { otherBoundaryLocalDofs.insert(it.getDofIndex()); interiorDofs.clear(); + + ERROR_EXIT("Rewrite the following code block!\n"); +#if 0 DofIndexToBool& isRankDof = meshDistributor->getIsRankDof(feSpace); for (DofIndexToBool::iterator dofIt = isRankDof.begin(); dofIt != isRankDof.end(); ++dofIt) { @@ -108,6 +111,7 @@ namespace AMDiS { otherBoundaryLocalDofs.count(dofIt->first) == 0) interiorDofs.insert(meshDistributor->mapDofToGlobal(feSpace, dofIt->first)); } +#endif nInteriorDofs = interiorDofs.size(); mpi::getDofNumbering(mpiComm, nInteriorDofs, -- GitLab