Commit f8e4f05c authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on parallel jump residuum. Is working, but source code must be improved.

parent 59a6d168
...@@ -389,24 +389,12 @@ namespace AMDiS { ...@@ -389,24 +389,12 @@ namespace AMDiS {
/// Sets \ref est_sum. /// Sets \ref est_sum.
inline void setEstSum(double e, int index) inline void setEstSum(double e, int index)
{ {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double send_est_sum = e;
double est_sum = 0.0;
MPI::COMM_WORLD.Allreduce(&send_est_sum, &est_sum, 1, MPI_DOUBLE, MPI_SUM);
e = est_sum;
#endif
scalContents[index]->est_sum = e; scalContents[index]->est_sum = e;
} }
/// Sets \ref est_max. /// Sets \ref est_max.
inline void setEstMax(double e, int index) inline void setEstMax(double e, int index)
{ {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double send_est_max = e;
double est_max = 0.0;
MPI::COMM_WORLD.Allreduce(&send_est_max, &est_max, 1, MPI_DOUBLE, MPI_MAX);
e = est_max;
#endif
scalContents[index]->est_max = e; scalContents[index]->est_max = e;
} }
......
...@@ -128,6 +128,19 @@ namespace AMDiS { ...@@ -128,6 +128,19 @@ namespace AMDiS {
} }
bool BoundaryObject::operator<(const BoundaryObject& other) const
{
if (elIndex == other.elIndex) {
if (subObj == other.subObj)
return ithObj < other.ithObj;
return subObj < other.subObj;
}
return (elIndex < other.elIndex);
}
bool AtomicBoundary::operator==(const AtomicBoundary& other) const bool AtomicBoundary::operator==(const AtomicBoundary& other) const
{ {
return (rankObj == other.rankObj && return (rankObj == other.rankObj &&
......
...@@ -55,6 +55,8 @@ namespace AMDiS { ...@@ -55,6 +55,8 @@ namespace AMDiS {
bool operator!=(const BoundaryObject& other) const; bool operator!=(const BoundaryObject& other) const;
bool operator<(const BoundaryObject& other) const;
/// The macro element to which the boundary element corresponds to. /// The macro element to which the boundary element corresponds to.
Element* el; Element* el;
...@@ -64,11 +66,9 @@ namespace AMDiS { ...@@ -64,11 +66,9 @@ namespace AMDiS {
/// Element type index, only used in 3d. /// Element type index, only used in 3d.
int elType; int elType;
/** \brief /// Defines the geometrical object at the boundary. It must be "a part" of
* Defines the geometrical object at the boundary. It must be "a part" of the /// the macro element \ref el, i.e., either 1 (a vertex), 2 (an edge) or 3
* macro element \ref el, i.e., either 1 (a vertex), 2 (an edge) or 3 /// (a face).
* (a face).
*/
GeoIndex subObj; GeoIndex subObj;
/** \brief /** \brief
......
...@@ -472,41 +472,29 @@ namespace AMDiS { ...@@ -472,41 +472,29 @@ namespace AMDiS {
/// \ref element is child of element parent /// \ref element is child of element parent
Element *parent; Element *parent;
/** \brief /// \ref element is an element of the binary tree located at MacroElement
* \ref element is an element of the binary tree located at MacroElement /// macroElement
* macroElement
*/
MacroElement *macroElement; MacroElement *macroElement;
/** \brief /// Indicates wich elements will be called and wich information should be
* Indicates wich elements will be called and wich information should be /// present while mesh traversal.
* present while mesh traversal.
*/
Flag fillFlag; Flag fillFlag;
/** \brief /// Level of the element. The level is zero for macro elements and the level
* Level of the element. The level is zero for macro elements and the level /// of the children is (level of the parent + 1). level_ is filled always by
* of the children is (level of the parent + 1). level_ is filled always by /// the traversal routines.
* the traversal routines.
*/
unsigned char level; unsigned char level;
/** \brief /// Elements type index. This is used only for 3d, where type can be either 0, 1 or
* Elements type index. This is used only for 3d, where type can be either 0, 1 or /// 2. In all other cases type is not used and the variable is set to 0.
* 2. In all other cases type is not used and the variable is set to 0. /// In 3d, it is filled automatically by the traversal routines.
* In 3d, it is filled automatically by the traversal routines.
*/
int elType; int elType;
/** \brief /// This ElInfo is the iChild-th child of the parent element.
* This ElInfo is the iChild-th child of the parent element.
*/
int iChild; int iChild;
/** \brief /// \ref coord[i] is a WorldVector<double> storing the world coordinates of the
* \ref coord[i] is a WorldVector<double> storing the world coordinates of the /// i-th vertex of element \ref element.
* i-th vertex of element \ref element.
*/
FixVec<WorldVector<double>, VERTEX> coord; FixVec<WorldVector<double>, VERTEX> coord;
/** \brief /** \brief
...@@ -518,34 +506,24 @@ namespace AMDiS { ...@@ -518,34 +506,24 @@ namespace AMDiS {
*/ */
FixVec<BoundaryType, BOUNDARY> boundary; FixVec<BoundaryType, BOUNDARY> boundary;
/** \brief /// Vector storing pointers to projections for each face, edge, vertex.
* Vector storing pointers to projections for each face, edge, vertex.
*/
FixVec<Projection*, PROJECTION> projection; FixVec<Projection*, PROJECTION> projection;
/** \brief /// oppCoord[i] coordinates of the i-th neighbour vertex opposite the
* oppCoord[i] coordinates of the i-th neighbour vertex opposite the /// common edge/face.
* common edge/face.
*/
FixVec<WorldVector<double>, NEIGH> oppCoord; FixVec<WorldVector<double>, NEIGH> oppCoord;
/** \brief /// neighbour[i] pointer to the element at the edge/face with local index i.
* neighbour[i] pointer to the element at the edge/face with local index i. /// It is a pointer to NULL for boundary edges/faces.
* It is a pointer to NULL for boundary edges/faces.
*/
FixVec<Element*, NEIGH> neighbour; FixVec<Element*, NEIGH> neighbour;
/** \brief /// neighbourCoord[i][j] are the coordinate of the j-th vertex of the i-th
* neighbourCoord[i][j] are the coordinate of the j-th vertex of the i-th /// neighbour element with the common edge/face.
* neighbour element with the common edge/face.
*/
FixVec<FixVec<WorldVector<double>, VERTEX>, NEIGH> neighbourCoord; FixVec<FixVec<WorldVector<double>, VERTEX>, NEIGH> neighbourCoord;
/** \brief /// oppVertex[i] is undefined if neighbour[i] is a pointer to NULL.
* oppVertex[i] is undefined if neighbour[i] is a pointer to NULL. /// Otherwise it is the local index of the neighbour's vertex opposite the
* Otherwise it is the local index of the neighbour's vertex opposite the /// common edge/face.
* common edge/face.
*/
FixVec<int, NEIGH> oppVertex; FixVec<int, NEIGH> oppVertex;
/// Elements determinant. /// Elements determinant.
...@@ -569,18 +547,14 @@ namespace AMDiS { ...@@ -569,18 +547,14 @@ namespace AMDiS {
static std::vector<std::map<std::pair<int, unsigned long>, mtl::dense2D<double> > > subElemGradMatrices; static std::vector<std::map<std::pair<int, unsigned long>, mtl::dense2D<double> > > subElemGradMatrices;
/** \brief /// child_vertex[el_type][child][i] = father's local vertex index of new
* child_vertex[el_type][child][i] = father's local vertex index of new /// vertex i. 4 stands for the newly generated vertex .
* vertex i. 4 stands for the newly generated vertex .
*/
static const int childVertex[3][2][4]; static const int childVertex[3][2][4];
/** \brief /// child_edge[el_type][child][i] = father's local edge index of new edge i.
* child_edge[el_type][child][i] = father's local edge index of new edge i. /// new edge 2 is half of old edge 0, new edges 4,5 are really new edges, and
* new edge 2 is half of old edge 0, new edges 4,5 are really new edges, and /// value is different: child_edge[][][4,5] = index of same edge in other
* value is different: child_edge[][][4,5] = index of same edge in other /// child.
* child.
*/
static const int childEdge[3][2][6]; static const int childEdge[3][2][6];
friend class ElInfo1d; friend class ElInfo1d;
......
...@@ -140,7 +140,7 @@ namespace AMDiS { ...@@ -140,7 +140,7 @@ namespace AMDiS {
markRLimit = ESThetaP * epsP / nLeaves; markRLimit = ESThetaP * epsP / nLeaves;
markCLimit = ESThetaCP * epsP / nLeaves; markCLimit = ESThetaCP * epsP / nLeaves;
INFO(info, 2)("start mark_limits: %.3le %.3le nt = %d\n", INFO(info, 2)("start mark_limits: %.3le %.3le nt = %d\n",
markRLimit, markCLimit, nLeaves); markRLimit, markCLimit, nLeaves);
} }
......
...@@ -1726,11 +1726,12 @@ namespace AMDiS { ...@@ -1726,11 +1726,12 @@ namespace AMDiS {
map<int, double> vec; map<int, double> vec;
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this->getMesh(comp), -1, ElInfo *elInfo =
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS); stack.traverseFirst(this->getMesh(comp), -1, Mesh::CALL_LEAF_EL);
while (elInfo) { while (elInfo) {
vec[elInfo->getElement()->getIndex()] = elInfo->getElement()->getEstimation(comp); vec[elInfo->getElement()->getIndex()] =
elInfo->getElement()->getEstimation(comp);
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
......
...@@ -157,19 +157,15 @@ namespace AMDiS { ...@@ -157,19 +157,15 @@ namespace AMDiS {
/** \brief /// Returns \ref lambda[a][b] which is the b-th coordinate entry of the a-th
* Returns \ref lambda[a][b] which is the b-th coordinate entry of the a-th /// quadrature point
* quadrature point
*/
inline double getLambda(int a, int b) const inline double getLambda(int a, int b) const
{ {
return (lambda ? (*lambda)[a][b] : 0.0); return (lambda ? (*lambda)[a][b] : 0.0);
} }
/** \brief /// Returns \ref lambda[a] which is a DimVec<double> containing the
* Returns \ref lambda[a] which is a DimVec<double> containing the /// coordiantes of the a-th quadrature point
* coordiantes of the a-th quadrature point
*/
inline const DimVec<double>& getLambda(int a) const inline const DimVec<double>& getLambda(int a) const
{ {
return (*lambda)[a]; return (*lambda)[a];
...@@ -206,10 +202,8 @@ namespace AMDiS { ...@@ -206,10 +202,8 @@ namespace AMDiS {
double *w; double *w;
protected: protected:
/** \brief /// Initialisation of all static Quadrature objects which will be returned
* Initialisation of all static Quadrature objects which will be returned /// by \ref provideQuadrature()
* by \ref provideQuadrature()
*/
static void initStaticQuadratures(); static void initStaticQuadratures();
/** \name static quadratures, used weights, and barycentric coords /** \name static quadratures, used weights, and barycentric coords
...@@ -288,21 +282,15 @@ namespace AMDiS { ...@@ -288,21 +282,15 @@ namespace AMDiS {
/** \brief /// Pre-compute the values of all basis functions at all quadrature nodes;
* Pre-compute the values of all basis functions at all quadrature nodes;
*/
const Flag INIT_PHI=1; const Flag INIT_PHI=1;
/** \brief /// Pre-compute the gradients (with respect to the barycentric coordinates) of
* Pre-compute the gradients (with respect to the barycentric coordinates) of /// all basis functions at all quadrature nodes
* all basis functions at all quadrature nodes
*/
const Flag INIT_GRD_PHI=2; const Flag INIT_GRD_PHI=2;
/** \brief /// pre-compute all 2nd derivatives (with respect to the barycentric
* pre-compute all 2nd derivatives (with respect to the barycentric /// coordinates) of all basis functions at all quadrature nodes;
* coordinates) of all basis functions at all quadrature nodes;
*/
const Flag INIT_D2_PHI=4; const Flag INIT_D2_PHI=4;
...@@ -325,10 +313,8 @@ namespace AMDiS { ...@@ -325,10 +313,8 @@ namespace AMDiS {
class FastQuadrature class FastQuadrature
{ {
protected: protected:
/** \brief /// Constructs a FastQuadrature for the given Quadrature, BasisFunction, and
* Constructs a FastQuadrature for the given Quadrature, BasisFunction, and /// flag.
* flag.
*/
FastQuadrature(BasisFunction* basFcts, Quadrature* quad, Flag flag) FastQuadrature(BasisFunction* basFcts, Quadrature* quad, Flag flag)
: init_flag(flag), : init_flag(flag),
phi(0, 0), phi(0, 0),
...@@ -481,10 +467,8 @@ namespace AMDiS { ...@@ -481,10 +467,8 @@ namespace AMDiS {
} }
protected: protected:
/** \brief /// Specifies which information should be pre-computed. Can be \ref INIT_PHI,
* Specifies which information should be pre-computed. Can be \ref INIT_PHI, /// \ref INIT_GRD_PHI, or \ref INIT_D2_PHI
* \ref INIT_GRD_PHI, or \ref INIT_D2_PHI
*/
Flag init_flag; Flag init_flag;
/** \brief /** \brief
...@@ -513,16 +497,12 @@ namespace AMDiS { ...@@ -513,16 +497,12 @@ namespace AMDiS {
*/ */
MatrixOfFixVecs<DimMat<double> > *D2Phi; MatrixOfFixVecs<DimMat<double> > *D2Phi;
/** \brief /// List of all used FastQuadratures
* List of all used FastQuadratures
*/
static list<FastQuadrature*> fastQuadList; static list<FastQuadrature*> fastQuadList;
/** \brief /// Maximal number of quadrature points for all yet initialised FastQuadrature
* Maximal number of quadrature points for all yet initialised FastQuadrature /// objects. This value may change after a new initialisation of a
* objects. This value may change after a new initialisation of a /// FastQuadrature
* FastQuadrature
*/
static int max_points; static int max_points;
/// This FastQuadrature stores values for Quadrature quadrature /// This FastQuadrature stores values for Quadrature quadrature
......
...@@ -173,7 +173,6 @@ namespace AMDiS { ...@@ -173,7 +173,6 @@ namespace AMDiS {
InteriorBoundary &intBoundary = InteriorBoundary &intBoundary =
MeshDistributor::globalMeshDistributor->getIntBoundary(); MeshDistributor::globalMeshDistributor->getIntBoundary();
RankToBoundMap& otherBound = intBoundary.getOther();
ElInfo *elInfo = mesh->createNewElInfo(); ElInfo *elInfo = mesh->createNewElInfo();
elInfo->setFillFlag(Mesh::FILL_COORDS); elInfo->setFillFlag(Mesh::FILL_COORDS);
...@@ -181,56 +180,66 @@ namespace AMDiS { ...@@ -181,56 +180,66 @@ namespace AMDiS {
StdMpi<vector<double> > stdMpiDet(MeshDistributor::globalMeshDistributor->getMpiComm()); StdMpi<vector<double> > stdMpiDet(MeshDistributor::globalMeshDistributor->getMpiComm());
StdMpi<vector<vector<WorldVector<double> > > > stdMpiGrdUh(MeshDistributor::globalMeshDistributor->getMpiComm()); StdMpi<vector<vector<WorldVector<double> > > > stdMpiGrdUh(MeshDistributor::globalMeshDistributor->getMpiComm());
for (RankToBoundMap::iterator it = otherBound.begin(); RankToBoundMap allBounds = intBoundary.getOther();
it != otherBound.end(); ++it) { allBounds.insert(intBoundary.getOwn().begin(), intBoundary.getOwn().end());
for (RankToBoundMap::iterator it = allBounds.begin();
it != allBounds.end(); ++it) {
vector<BoundaryObject> subBound;
for (unsigned int i = 0; i < it->second.size(); i++) { for (unsigned int i = 0; i < it->second.size(); i++) {
BoundaryObject &bObj = it->second[i].rankObj; BoundaryObject &bObj = it->second[i].rankObj;
if (bObj.subObj == VERTEX) if (bObj.subObj == VERTEX)
continue; continue;
vector<BoundaryObject> subBound;
bObj.el->getSubBoundary(bObj, subBound); bObj.el->getSubBoundary(bObj, subBound);
}
WorldVector<int> faceIndices; if (subBound.size() == 0)
continue;
for (unsigned int j = 0; j < subBound.size(); j++) {
Element *el = subBound[j].el;
int oppV = subBound[j].ithObj;
elInfo->setElement(el);
el->sortFaceIndices(oppV, faceIndices);
elInfo->getCoord(oppV) = coords[el->getDof(oppV, 0)];
for (int k = 0; k < dim; k++)
elInfo->getCoord(faceIndices[k]) = coords[el->getDof(k, 0)];
double detNeigh = abs(elInfo->calcGrdLambda(*lambdaNeigh));
stdMpiDet.getSendData(it->first).push_back(detNeigh);
for (int system = 0; system < nSystems; system++) {
if (matrix[system] == NULL || secondOrderTerms[system] == false)
continue;
uh[system]->getLocalVector(el, uhNeigh[system]);
for (int iq = 0; iq < nPointsSurface; iq++) {
(*lambda)[oppV] = 0.0; WorldVector<int> faceIndices;
for (int k = 0; k < dim; k++)
(*lambda)[faceIndices[k]] = surfaceQuad->getLambda(iq, k);
basFcts[system]->evalGrdUh(*lambda, *lambdaNeigh, uhNeigh[system], grdUhNeigh[iq]); for (unsigned int i = 0; i < subBound.size(); i++) {
} Element *el = subBound[i].el;
int oppV = subBound[i].ithObj;
elInfo->setElement(el);
el->sortFaceIndices(oppV, faceIndices);
for (int k = 0; k <= dim; k++)
elInfo->getCoord(k) = coords[el->getDof(k, 0)];
double detNeigh = abs(elInfo->calcGrdLambda(*lambdaNeigh));
stdMpiDet.getSendData(it->first).push_back(detNeigh);
stdMpiGrdUh.getSendData(it->first).push_back(grdUhNeigh); for (int system = 0; system < nSystems; system++) {
if (matrix[system] == NULL || secondOrderTerms[system] == false)
continue;
uh[system]->getLocalVector(el, uhNeigh[system]);
for (int iq = 0; iq < nPointsSurface; iq++) {
(*lambda)[oppV] = 0.0;
for (int k = 0; k < dim; k++)
(*lambda)[faceIndices[k]] = surfaceQuad->getLambda(iq, k);
basFcts[system]->evalGrdUh(*lambda, *lambdaNeigh, uhNeigh[system], grdUhNeigh[iq]);
} }
stdMpiGrdUh.getSendData(it->first).push_back(grdUhNeigh);
} }
} }
stdMpiDet.recv(it->first);
stdMpiGrdUh.recv(it->first);
} }
stdMpiDet.updateSendDataSize(); stdMpiDet.updateSendDataSize();
stdMpiGrdUh.updateSendDataSize(); stdMpiGrdUh.updateSendDataSize();
#if 0
RankToBoundMap& ownBound = intBoundary.getOwn(); RankToBoundMap& ownBound = intBoundary.getOwn();
for (RankToBoundMap::iterator it = ownBound.begin(); for (RankToBoundMap::iterator it = ownBound.begin();
...@@ -245,13 +254,14 @@ namespace AMDiS { ...@@ -245,13 +254,14 @@ namespace AMDiS {
} }
} }
} }
#endif
stdMpiDet.startCommunication(); stdMpiDet.startCommunication();
stdMpiGrdUh.startCommunication(); stdMpiGrdUh.startCommunication();
for (RankToBoundMap::iterator it = ownBound.begin(); for (RankToBoundMap::iterator it = allBounds.begin();
it != ownBound.end(); ++it) { it != allBounds.end(); ++it) {
vector<BoundaryObject> subBound; vector<BoundaryObject> subBound;
for (unsigned int i = 0; i < it->second.size(); i++) { for (unsigned int i = 0; i < it->second.size(); i++) {
...@@ -270,6 +280,11 @@ namespace AMDiS { ...@@ -270,6 +280,11 @@ namespace AMDiS {
TEST_EXIT_DBG(subBound.size() == stdMpiGrdUh.getRecvData(it->first).size()) TEST_EXIT_DBG(subBound.size() == stdMpiGrdUh.getRecvData(it->first).size())
("Should not happen!\n"); ("Should not happen!\n");
for (unsigned int i = 0; i < subBound.size(); i++) {
elBoundDet[subBound[i]] = stdMpiDet.getRecvData(it->first)[i];
elBoundGrdUhNeigh[subBound[i]] = stdMpiGrdUh.getRecvData(it->first)[i];
}
} }
MSG("INIT PARALLEL FINISCHED!\n"); MSG("INIT PARALLEL FINISCHED!\n");
...@@ -482,20 +497,41 @@ namespace AMDiS { ...@@ -482,20 +497,41 @@ namespace AMDiS {
const DimVec<WorldVector<double> > &grdLambda = elInfo->getGrdLambda(); const DimVec<WorldVector<double> > &grdLambda = elInfo->getGrdLambda();
double det = elInfo->getDet(); double det = elInfo->getDet();
double h2 = h2_from_det(det, dim); double h2 = h2_from_det(det, dim);
BoundaryObject blub;
for (int face = 0; face < nNeighbours; face++) { for (int face = 0; face < nNeighbours; face++) {
Element *neigh = const_cast<Element*>(elInfo->getNeighbour(face)); Element *neigh = const_cast<Element*>(elInfo->getNeighbour(face));
bool parallelMode = false;
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (neigh == NULL) {
BoundaryObject testObj(el, elInfo->getType(), EDGE, face);
if (elBoundDet.count(testObj)) {
parallelMode = true;
blub = testObj;