From 531be06cf9bb0740ef142e874058c097cde90ce0 Mon Sep 17 00:00:00 2001 From: Thomas Witkowski Date: Fri, 27 Apr 2012 16:27:43 +0000 Subject: [PATCH] Some large changes in code, seems to work.... --- AMDiS/src/AMDiS_fwd.h | 1 + AMDiS/src/parallel/DofComm.cc | 55 +- AMDiS/src/parallel/DofComm.h | 16 +- AMDiS/src/parallel/ElementObjectDatabase.cc | 101 ++-- AMDiS/src/parallel/ElementObjectDatabase.h | 103 ++-- AMDiS/src/parallel/InteriorBoundary.cc | 359 +++++++++++-- AMDiS/src/parallel/InteriorBoundary.h | 61 +-- AMDiS/src/parallel/MeshDistributor.cc | 529 +++----------------- AMDiS/src/parallel/MeshDistributor.h | 52 +- AMDiS/src/parallel/ParallelDebug.cc | 48 +- AMDiS/src/parallel/ParallelTypes.h | 6 +- 11 files changed, 641 insertions(+), 690 deletions(-) diff --git a/AMDiS/src/AMDiS_fwd.h b/AMDiS/src/AMDiS_fwd.h index 994e8ff8..a0627ae3 100644 --- a/AMDiS/src/AMDiS_fwd.h +++ b/AMDiS/src/AMDiS_fwd.h @@ -98,6 +98,7 @@ namespace AMDiS { class VertexVector; #ifdef HAVE_PARALLEL_DOMAIN_AMDIS + class ElementObjectDatabase; class FeSpaceDofMap; class MeshLevelData; #endif diff --git a/AMDiS/src/parallel/DofComm.cc b/AMDiS/src/parallel/DofComm.cc index 28fa6378..6507ab23 100644 --- a/AMDiS/src/parallel/DofComm.cc +++ b/AMDiS/src/parallel/DofComm.cc @@ -10,15 +10,45 @@ // See also license.opensource.txt in the distribution. -#include "DofComm.h" +#include "parallel/DofComm.h" +#include "parallel/InteriorBoundary.h" +#include "FiniteElemSpace.h" namespace AMDiS { using namespace std; - void DofComm::removeEmpty() + int DofComm::getNumberDofs(int level, const FiniteElemSpace *feSpace) { - FUNCNAME("DofComm::removeEmpty()"); + FUNCNAME("DofComm::getNumberDofs()"); + + TEST_EXIT_DBG(level < data.size())("Should not happen!\n"); + + DofContainerSet dofs; + + for (DataIter rankIt = data[level].begin(); + rankIt != data[level].end(); ++rankIt) + for (FeMapIter feIt = rankIt->second.begin(); + feIt != rankIt->second.end(); ++feIt) + if (feIt->first == feSpace) + dofs.insert(feIt->second.begin(), feIt->second.end()); + + return static_cast(dofs.size()); + } + + + void DofComm::create(RankToBoundMap &boundary) + { + // === Fill data. === + + for (unsigned int i = 0; i < feSpaces.size(); i++) + for (int level = 0; level < nLevel; level++) + for (InteriorBoundary::iterator it(boundary, level); !it.end(); ++it) + it->rankObj.el->getAllDofs(feSpaces[i], it->rankObj, + data[level][it.getRank()][feSpaces[i]]); + + + // === Remove empty data containers. === for (unsigned int i = 0; i < data.size(); i++) { DataIter dit = data[i].begin(); @@ -42,25 +72,6 @@ namespace AMDiS { } - int DofComm::getNumberDofs(int level, const FiniteElemSpace *feSpace) - { - FUNCNAME("DofComm::getNumberDofs()"); - - TEST_EXIT_DBG(level < data.size())("Should not happen!\n"); - - DofContainerSet dofs; - - for (DataIter rankIt = data[level].begin(); - rankIt != data[level].end(); ++rankIt) - for (FeMapIter feIt = rankIt->second.begin(); - feIt != rankIt->second.end(); ++feIt) - if (feIt->first == feSpace) - dofs.insert(feIt->second.begin(), feIt->second.end()); - - return static_cast(dofs.size()); - } - - bool DofComm::Iterator::setNextFeMap() { FUNCNAME("DofComm::Iterator::setNextFeMap()"); diff --git a/AMDiS/src/parallel/DofComm.h b/AMDiS/src/parallel/DofComm.h index 6e5467a1..89874a4b 100644 --- a/AMDiS/src/parallel/DofComm.h +++ b/AMDiS/src/parallel/DofComm.h @@ -24,6 +24,7 @@ #define AMDIS_DOF_COMM_H #include +#include "parallel/ParallelTypes.h" #include "FiniteElemSpace.h" #include "Global.h" @@ -52,12 +53,13 @@ namespace AMDiS { return data[level][rank][feSpace]; } - void removeEmpty(); - - void init(int nLevels = 1) + void init(int n, vector &fe) { + nLevel = n; + feSpaces = fe; + data.clear(); - data.resize(nLevels); + data.resize(nLevel); } DataType& getData(int level = 0) @@ -67,9 +69,15 @@ namespace AMDiS { int getNumberDofs(int level, const FiniteElemSpace *feSpace); + void create(RankToBoundMap &boundary); + protected: LevelDataType data; + int nLevel; + + vector feSpaces; + friend class Iterator; public: diff --git a/AMDiS/src/parallel/ElementObjectDatabase.cc b/AMDiS/src/parallel/ElementObjectDatabase.cc index 7f6cc65c..54d84b48 100644 --- a/AMDiS/src/parallel/ElementObjectDatabase.cc +++ b/AMDiS/src/parallel/ElementObjectDatabase.cc @@ -15,6 +15,49 @@ namespace AMDiS { + void ElementObjectDatabase::create() + { + // === Fills macro element data structures. === + + TraverseStack stack; + ElInfo *elInfo = stack.traverseFirst(mesh, -1, + Mesh::CALL_LEAF_EL | + Mesh::FILL_NEIGH | + Mesh::FILL_BOUND); + while (elInfo) { + TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n"); + + Element *el = elInfo->getElement(); + macroElIndexMap.insert(make_pair(el->getIndex(), el)); + macroElIndexTypeMap.insert(make_pair(el->getIndex(), elInfo->getType())); + + // Add all sub object of the element to the variable elObjDb. + addElement(elInfo); + + elInfo = stack.traverseNext(elInfo); + } + + // Create periodic data, if there are periodic boundary conditions. + createPeriodicData(); + + // Create data about the reverse modes of neighbouring elements. + createReverseModeData(); + } + + + void ElementObjectDatabase::createMacroElementInfo(vector &mel) + { + macroElIndexMap.clear(); + macroElIndexTypeMap.clear(); + + for (vector::iterator it = mel.begin(); + it != mel.end(); ++it) { + macroElIndexMap.insert(make_pair((*it)->getIndex(), (*it)->getElement())); + macroElIndexTypeMap.insert(make_pair((*it)->getIndex(), (*it)->getElType())); + } + } + + void ElementObjectDatabase::addElement(ElInfo *elInfo) { FUNCNAME("ElementObjectDatabase::addElement()"); @@ -128,7 +171,7 @@ namespace AMDiS { } - void ElementObjectDatabase::createPeriodicData(const FiniteElemSpace *feSpace) + void ElementObjectDatabase::createPeriodicData() { FUNCNAME("ElementObjectDatabase::createPeriodicData()"); @@ -223,9 +266,9 @@ namespace AMDiS { ("Should not happen!\n"); periodicVertices[make_pair(dof0, dof3)] = - provideConnectedPeriodicBoundary(feSpace->getAdmin(), type0, type1); + provideConnectedPeriodicBoundary(type0, type1); periodicVertices[make_pair(dof3, dof0)] = - provideConnectedPeriodicBoundary(feSpace->getAdmin(), type0, type1); + provideConnectedPeriodicBoundary(type0, type1); for (unsigned int j = i + 1; j < multPeriodicDof2.size(); j++) if (multPeriodicDof2[j] == dof3) @@ -244,7 +287,7 @@ namespace AMDiS { make_pair(multPeriodicDof3[j], multPeriodicDof3[i]); if (periodicVertices.count(perDofs0) == 0) { - BoundaryType b = getNewBoundaryType(feSpace->getAdmin()); + BoundaryType b = getNewBoundaryType(); periodicVertices[perDofs0] = b; periodicVertices[perDofs1] = b; } @@ -275,8 +318,7 @@ namespace AMDiS { BoundaryType type0 = periodicEdges[make_pair(edge0, edge1)]; BoundaryType type1 = periodicEdges[make_pair(edge0, edge2)]; - BoundaryType type2 = - provideConnectedPeriodicBoundary(feSpace->getAdmin(), type0, type1); + BoundaryType type2 = provideConnectedPeriodicBoundary(type0, type1); periodicEdges[perEdge0] = type2; periodicEdges[perEdge1] = type2; @@ -321,7 +363,7 @@ namespace AMDiS { } - BoundaryType ElementObjectDatabase::getNewBoundaryType(DOFAdmin *admin) + BoundaryType ElementObjectDatabase::getNewBoundaryType() { FUNCNAME("ElementObjectDatabase::getNewBoundaryType()"); @@ -334,15 +376,14 @@ namespace AMDiS { newPeriodicBoundaryType--; mesh->getPeriodicAssociations()[newPeriodicBoundaryType] = - new VertexVector(admin, ""); + new VertexVector(feSpace->getAdmin(), ""); return newPeriodicBoundaryType; } BoundaryType - ElementObjectDatabase::provideConnectedPeriodicBoundary(DOFAdmin *admin, - BoundaryType b0, + ElementObjectDatabase::provideConnectedPeriodicBoundary(BoundaryType b0, BoundaryType b1) { FUNCNAME("ElementObjectDatabase::provideConnectedPeriodicBoundary()"); @@ -351,13 +392,13 @@ namespace AMDiS { (b0 <= b1 ? make_pair(b0, b1) : make_pair(b1, b0)); if (bConnMap.count(bConn) == 0) { - BoundaryType newPeriodicBoundaryType = getNewBoundaryType(admin); + BoundaryType newPeriodicBoundaryType = getNewBoundaryType(); VertexVector &vecB0 = mesh->getPeriodicAssociations(b0); VertexVector &vecB1 = mesh->getPeriodicAssociations(b1); VertexVector &vecC = mesh->getPeriodicAssociations(newPeriodicBoundaryType); - DOFIteratorBase it(const_cast(admin), USED_DOFS); + DOFIteratorBase it(const_cast(feSpace->getAdmin()), USED_DOFS); for (it.reset(); !it.end(); ++it) { if (!it.isDofFree()) { @@ -478,9 +519,7 @@ namespace AMDiS { } - void ElementObjectDatabase::createReverseModeData(const FiniteElemSpace* feSpace, - map &elIndexMap, - map &elIndexTypeMap) + void ElementObjectDatabase::createReverseModeData() { FUNCNAME("ElementObjectDatabase::createReverseModeData()"); @@ -498,13 +537,13 @@ namespace AMDiS { vector& els = edgeIt->second; for (unsigned int i = 0; i < els.size(); i++) { - BoundaryObject obj0(elIndexMap[els[i].elIndex], - elIndexTypeMap[els[i].elIndex], + BoundaryObject obj0(macroElIndexMap[els[i].elIndex], + macroElIndexTypeMap[els[i].elIndex], EDGE, els[i].ithObject); for (unsigned int j = i + 1; j < els.size(); j++) { - BoundaryObject obj1(elIndexMap[els[j].elIndex], - elIndexTypeMap[els[j].elIndex], + BoundaryObject obj1(macroElIndexMap[els[j].elIndex], + macroElIndexTypeMap[els[j].elIndex], EDGE, els[j].ithObject); bool reverseMode = @@ -522,13 +561,13 @@ namespace AMDiS { vector& els = faceIt->second; for (unsigned int i = 0; i < els.size(); i++) { - BoundaryObject obj0(elIndexMap[els[i].elIndex], - elIndexTypeMap[els[i].elIndex], + BoundaryObject obj0(macroElIndexMap[els[i].elIndex], + macroElIndexTypeMap[els[i].elIndex], FACE, els[i].ithObject); for (unsigned int j = i + 1; j < els.size(); j++) { - BoundaryObject obj1(elIndexMap[els[j].elIndex], - elIndexTypeMap[els[j].elIndex], + BoundaryObject obj1(macroElIndexMap[els[j].elIndex], + macroElIndexTypeMap[els[j].elIndex], FACE, els[j].ithObject); bool reverseMode = @@ -549,13 +588,13 @@ namespace AMDiS { vector &edges1 = edgeElements[edgeIt->first.second]; for (unsigned int i = 0; i < edges0.size(); i++) { - BoundaryObject obj0(elIndexMap[edges0[i].elIndex], - elIndexTypeMap[edges0[i].elIndex], + BoundaryObject obj0(macroElIndexMap[edges0[i].elIndex], + macroElIndexTypeMap[edges0[i].elIndex], EDGE, edges0[i].ithObject); for (unsigned int j = 0; j < edges1.size(); j++) { - BoundaryObject obj1(elIndexMap[edges1[j].elIndex], - elIndexTypeMap[edges1[j].elIndex], + BoundaryObject obj1(macroElIndexMap[edges1[j].elIndex], + macroElIndexTypeMap[edges1[j].elIndex], EDGE, edges1[j].ithObject); bool reverseMode = @@ -575,11 +614,11 @@ namespace AMDiS { TEST_EXIT_DBG(faces0.size() == faces1.size() == 1)("Should not happen!\n"); - BoundaryObject obj0(elIndexMap[faces0[0].elIndex], - elIndexTypeMap[faces0[0].elIndex], + BoundaryObject obj0(macroElIndexMap[faces0[0].elIndex], + macroElIndexTypeMap[faces0[0].elIndex], FACE, faces0[0].ithObject); - BoundaryObject obj1(elIndexMap[faces1[0].elIndex], - elIndexTypeMap[faces1[0].elIndex], + BoundaryObject obj1(macroElIndexMap[faces1[0].elIndex], + macroElIndexTypeMap[faces1[0].elIndex], FACE, faces1[0].ithObject); bool reverseMode = diff --git a/AMDiS/src/parallel/ElementObjectDatabase.h b/AMDiS/src/parallel/ElementObjectDatabase.h index 051e57bc..963f4c28 100644 --- a/AMDiS/src/parallel/ElementObjectDatabase.h +++ b/AMDiS/src/parallel/ElementObjectDatabase.h @@ -103,39 +103,25 @@ namespace AMDiS { class ElementObjectDatabase { public: ElementObjectDatabase() - : mesh(NULL), + : feSpace(NULL), + mesh(NULL), iterGeoPos(CENTER) {} - - /// Set the mesh that should be used for the database. - void setMesh(Mesh *m) + void setFeSpace(const FiniteElemSpace *fe) { - mesh = m; + feSpace = fe; + mesh = feSpace->getMesh(); + } + + Mesh* getMesh() + { + return mesh; } + void create(); - /** \brief - * Adds an element to the object database. If the element is part of a - * periodic boundary, all information about subobjects of the element on - * this boundary are collected. - * - * \param[in] elInfo ElInfo object of the element. - */ - void addElement(ElInfo *elInfo); - - - /** \brief - * Creates final data of the periodic boundaries. Must be called after all - * elements of the mesh are added to the object database. Then this functions - * search for indirectly connected vertices in periodic boundaries. This is - * only the case, if there are more than one boundary conditions. Then, e.g., - * in 2D, all edges of a square are iterectly connected. In 3D, if the macro - * mesh is a box, all eight vertex nodes and always four of the 12 edges are - * indirectly connected. - */ - void createPeriodicData(const FiniteElemSpace *feSpace); - + void createMacroElementInfo(vector &mel); /** \brief * Create for a filled object database the membership information for all @@ -149,22 +135,6 @@ namespace AMDiS { MeshLevelData& levelData); - /** \brief - * Creates on all boundaries the reverse mode flag. - * - * \param[in] feSpace An arbitrary FE space defined on the mesh. - * Is used to get the orientation of the DOFs on - * elements. - * \param[in] elIndexMap Maps an element index to the pointer to the - * element. - * \param[in] elIndexTypeMap Maps an element index to its type id (only - * relevant in 3D). - */ - void createReverseModeData(const FiniteElemSpace* feSpace, - map &elIndexMap, - map &elIndexTypeMap); - - /** \brief * Iterates over all elements for one geometrical index, i.e., over all * vertices, edges or faces in the mesh. The function returns true, if the @@ -480,6 +450,16 @@ namespace AMDiS { return (t >= smallestPeriodicBcType); } + inline Element* getElementPtr(int index) + { + return macroElIndexMap[index]; + } + + inline int getElementType(int index) + { + return macroElIndexTypeMap[index]; + } + /// Write the element database to disk. void serialize(ostream &out); @@ -487,6 +467,15 @@ namespace AMDiS { void deserialize(istream &in); protected: + /** \brief + * Adds an element to the object database. If the element is part of a + * periodic boundary, all information about subobjects of the element on + * this boundary are collected. + * + * \param[in] elInfo ElInfo object of the element. + */ + void addElement(ElInfo *elInfo); + /// Adds the i-th DOF vertex of an element to the object database. void addVertex(Element *el, int ith) { @@ -520,10 +509,24 @@ namespace AMDiS { faceLocalMap[elObj] = face; } - BoundaryType getNewBoundaryType(DOFAdmin *admin); + /** \brief + * Creates final data of the periodic boundaries. Must be called after all + * elements of the mesh are added to the object database. Then this functions + * search for indirectly connected vertices in periodic boundaries. This is + * only the case, if there are more than one boundary conditions. Then, e.g., + * in 2D, all edges of a square are iterectly connected. In 3D, if the macro + * mesh is a box, all eight vertex nodes and always four of the 12 edges are + * indirectly connected. + */ + void createPeriodicData(); + - BoundaryType provideConnectedPeriodicBoundary(DOFAdmin *admin, - BoundaryType b0, + /// Creates on all boundaries the reverse mode flag. + void createReverseModeData(); + + BoundaryType getNewBoundaryType(); + + BoundaryType provideConnectedPeriodicBoundary(BoundaryType b0, BoundaryType b1); /// Some auxiliary function to write the element object database to disk. @@ -539,10 +542,12 @@ namespace AMDiS { void deserialize(istream &in, map& data); private: + const FiniteElemSpace* feSpace; + /// The mesh that is used to store all its element information in /// the database. Mesh *mesh; - + /// Maps to each vertex DOF all element objects that represent this vertex. map > vertexElements; @@ -630,6 +635,12 @@ namespace AMDiS { map, bool> edgeReverseMode; map, bool> faceReverseMode; + + /// Maps to each macro element index a pointer to the corresponding element. + map macroElIndexMap; + + /// Maps to each macro element index the type of this element. + map macroElIndexTypeMap; }; } diff --git a/AMDiS/src/parallel/InteriorBoundary.cc b/AMDiS/src/parallel/InteriorBoundary.cc index 3dc98167..abee4b20 100644 --- a/AMDiS/src/parallel/InteriorBoundary.cc +++ b/AMDiS/src/parallel/InteriorBoundary.cc @@ -11,6 +11,7 @@ #include "parallel/InteriorBoundary.h" +#include "parallel/ElementObjectDatabase.h" #include "FiniteElemSpace.h" #include "BasisFunction.h" #include "Serializer.h" @@ -18,50 +19,352 @@ namespace AMDiS { - AtomicBoundary& InteriorBoundary::getNewAtomic(int rank) + AtomicBoundary& InteriorBoundary::getNewAtomicOwn(int rank) { - int size = boundary[rank].size(); - boundary[rank].resize(size + 1); - return boundary[rank][size]; + int size = own[rank].size(); + own[rank].resize(size + 1); + return own[rank][size]; } + AtomicBoundary& InteriorBoundary::getNewAtomicOther(int rank) + { + int size = other[rank].size(); + other[rank].resize(size + 1); + return other[rank][size]; + } - bool InteriorBoundary::operator==(const InteriorBoundary& other) const + AtomicBoundary& InteriorBoundary::getNewAtomicPer(int rank) { - InteriorBoundary& other2 = const_cast(other); + int size = periodic[rank].size(); + periodic[rank].resize(size + 1); + return periodic[rank][size]; + } + + + void InteriorBoundary::create(MPI::Intracomm &mpiComm, + ElementObjectDatabase &elObjDb) + { + FUNCNAME("InteriorBoundary::clear()"); + + own.clear(); + other.clear(); + periodic.clear(); + + Mesh *mesh = elObjDb.getMesh(); + TEST_EXIT_DBG(mesh)("Should not happen!\n"); + + int mpiRank = mpiComm.Get_rank(); - if (boundary.size() != other2.boundary.size()) - return false; + // === Create interior boundary data structure. === + + for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) { + GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim()); - for (unsigned int level = 0; level < boundary.size(); level++) { - for (RankToBoundMap::const_iterator it = boundary.begin(); - it != boundary.end(); ++it) { - if (other2.boundary.count(it->first) == 0) - return false; + while (elObjDb.iterate(geoIndex)) { + map& objData = elObjDb.getIterateData(); + if (!(objData.count(mpiRank) && objData.size() > 1)) + continue; + + int owner = elObjDb.getIterateOwner(); + ElementObjectData& rankBoundEl = objData[mpiRank]; + + AtomicBoundary bound; + bound.maxLevel = elObjDb.getIterateMaxLevel(); + + bound.rankObj.el = elObjDb.getElementPtr(rankBoundEl.elIndex); + bound.rankObj.elIndex = rankBoundEl.elIndex; + bound.rankObj.elType = elObjDb.getElementType(rankBoundEl.elIndex); + bound.rankObj.subObj = geoIndex; + bound.rankObj.ithObj = rankBoundEl.ithObject; + + if (geoIndex == FACE) { + for (int edgeNo = 0; edgeNo < 3; edgeNo++) { + int edgeOfFace = + bound.rankObj.el->getEdgeOfFace(bound.rankObj.ithObj, edgeNo); + + bound.rankObj.excludedSubstructures.push_back(make_pair(EDGE, edgeOfFace)); + } + } - if (other2.boundary[it->first].size() != it->second.size()) - return false; - for (unsigned int i = 0; i < it->second.size(); i++) { - std::vector::iterator bIt = - find(other2.boundary[it->first].begin(), - other2.boundary[it->first].end(), it->second[i]); + if (owner == mpiRank) { + for (map::iterator it2 = objData.begin(); + it2 != objData.end(); ++it2) { + if (it2->first == mpiRank) + continue; + + bound.neighObj.el = elObjDb.getElementPtr(it2->second.elIndex); + bound.neighObj.elIndex = it2->second.elIndex; + bound.neighObj.elType = elObjDb.getElementType(it2->second.elIndex); + bound.neighObj.subObj = geoIndex; + bound.neighObj.ithObj = it2->second.ithObject; + + bound.type = INTERIOR; + + AtomicBoundary& b = getNewAtomicOwn(it2->first); + b = bound; + if (geoIndex == EDGE) + b.neighObj.reverseMode = + elObjDb.getEdgeReverseMode(rankBoundEl, it2->second); + if (geoIndex == FACE) + b.neighObj.reverseMode = + elObjDb.getFaceReverseMode(rankBoundEl, it2->second); + } + + } else { + TEST_EXIT_DBG(objData.count(owner) == 1) + ("Should not happen!\n"); + + ElementObjectData& ownerBoundEl = objData[owner]; - if (bIt == other2.boundary[it->first].end()) - return false; - } + bound.neighObj.el = elObjDb.getElementPtr(ownerBoundEl.elIndex); + bound.neighObj.elIndex = ownerBoundEl.elIndex; + bound.neighObj.elType = -1; + bound.neighObj.subObj = geoIndex; + bound.neighObj.ithObj = ownerBoundEl.ithObject; + + bound.type = INTERIOR; + + AtomicBoundary& b = getNewAtomicOther(owner); + b = bound; + if (geoIndex == EDGE) + b.rankObj.reverseMode = + elObjDb.getEdgeReverseMode(rankBoundEl, ownerBoundEl); + if (geoIndex == FACE) + b.rankObj.reverseMode = + elObjDb.getFaceReverseMode(rankBoundEl, ownerBoundEl); + } } } - return true; - } + // === Create periodic boundary data structure. === - void InteriorBoundary::clear() - { - FUNCNAME("InteriorBoundary::clear()"); + for (PerBoundMap::iterator it = elObjDb.getPeriodicVertices().begin(); + it != elObjDb.getPeriodicVertices().end(); ++it) { + if (elObjDb.isInRank(it->first.first, mpiRank) == false) + continue; + + ElementObjectData& perDofEl0 = + elObjDb.getElementsInRank(it->first.first)[mpiRank]; + + for (map::iterator elIt = elObjDb.getElementsInRank(it->first.second).begin(); + elIt != elObjDb.getElementsInRank(it->first.second).end(); ++elIt) { + + int otherElementRank = elIt->first; + ElementObjectData& perDofEl1 = elIt->second; + + AtomicBoundary bound; + bound.rankObj.el = elObjDb.getElementPtr(perDofEl0.elIndex); + bound.rankObj.elIndex = perDofEl0.elIndex; + bound.rankObj.elType = elObjDb.getElementType(perDofEl0.elIndex); + bound.rankObj.subObj = VERTEX; + bound.rankObj.ithObj = perDofEl0.ithObject; + + bound.neighObj.el = elObjDb.getElementPtr(perDofEl1.elIndex); + bound.neighObj.elIndex = perDofEl1.elIndex; + bound.neighObj.elType = elObjDb.getElementType(perDofEl1.elIndex); + bound.neighObj.subObj = VERTEX; + bound.neighObj.ithObj = perDofEl1.ithObject; + + bound.type = it->second; + + AtomicBoundary& b = getNewAtomicPer(otherElementRank); + b = bound; + } + } + + + for (PerBoundMap::iterator it = elObjDb.getPeriodicEdges().begin(); + it != elObjDb.getPeriodicEdges().end(); ++it) { + if (elObjDb.isInRank(it->first.first, mpiRank) == false) + continue; + + ElementObjectData& perEdgeEl0 = elObjDb.getElementsInRank(it->first.first)[mpiRank]; + + for (map::iterator elIt = elObjDb.getElementsInRank(it->first.second).begin(); + elIt != elObjDb.getElementsInRank(it->first.second).end(); ++elIt) { + + int otherElementRank = elIt->first; + ElementObjectData& perEdgeEl1 = elIt->second; + + AtomicBoundary bound; + bound.rankObj.el = elObjDb.getElementPtr(perEdgeEl0.elIndex); + bound.rankObj.elIndex = perEdgeEl0.elIndex; + bound.rankObj.elType = elObjDb.getElementType(perEdgeEl0.elIndex); + bound.rankObj.subObj = EDGE; + bound.rankObj.ithObj = perEdgeEl0.ithObject; + + bound.neighObj.el = elObjDb.getElementPtr(perEdgeEl1.elIndex); + bound.neighObj.elIndex = perEdgeEl1.elIndex; + bound.neighObj.elType = elObjDb.getElementType(perEdgeEl1.elIndex); + bound.neighObj.subObj = EDGE; + bound.neighObj.ithObj = perEdgeEl1.ithObject; + + bound.type = it->second; + + AtomicBoundary& b = getNewAtomicPer(otherElementRank); + b = bound; + + if (mpiRank > otherElementRank) + b.neighObj.reverseMode = + elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); + else + b.rankObj.reverseMode = + elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); + } + } + + + for (PerBoundMap::iterator it = elObjDb.getPeriodicFaces().begin(); + it != elObjDb.getPeriodicFaces().end(); ++it) { + if (elObjDb.isInRank(it->first.first, mpiRank) == false) + continue; + + TEST_EXIT_DBG(elObjDb.getElements(it->first.first).size() == 1) + ("Should not happen!\n"); + TEST_EXIT_DBG(elObjDb.getElements(it->first.second).size() == 1) + ("Should not happen!\n"); + + ElementObjectData& perFaceEl0 = elObjDb.getElementsInRank(it->first.first)[mpiRank]; + + for (map::iterator elIt = elObjDb.getElementsInRank(it->first.second).begin(); + elIt != elObjDb.getElementsInRank(it->first.second).end(); ++elIt) { + + int otherElementRank = elIt->first; + ElementObjectData& perFaceEl1 = elIt->second; + + AtomicBoundary bound; + bound.rankObj.el = elObjDb.getElementPtr(perFaceEl0.elIndex); + bound.rankObj.elIndex = perFaceEl0.elIndex; + bound.rankObj.elType = elObjDb.getElementType(perFaceEl0.elIndex); + bound.rankObj.subObj = FACE; + bound.rankObj.ithObj = perFaceEl0.ithObject; + + bound.neighObj.el = elObjDb.getElementPtr(perFaceEl1.elIndex); + bound.neighObj.elIndex = perFaceEl1.elIndex; + bound.neighObj.elType = elObjDb.getElementType(perFaceEl1.elIndex); + bound.neighObj.subObj = FACE; + bound.neighObj.ithObj = perFaceEl1.ithObject; + + bound.type = it->second; + + AtomicBoundary& b = getNewAtomicPer(otherElementRank); + b = bound; + + if (mpiRank > otherElementRank) + b.neighObj.reverseMode = + elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1); + else + b.rankObj.reverseMode = + elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1); + } + } + + + // === Once we have this information, we must care about the order of the === + // === atomic bounds in the three boundary handling object. Eventually === + // === all the boundaries have to be in the same order on both ranks that === + // === share the bounday. === + + StdMpi > stdMpi(mpiComm); + stdMpi.send(own); + stdMpi.recv(other); + stdMpi.startCommunication(); + + + // === The information about all neighbouring boundaries has been === + // === received. So the rank tests if its own atomic boundaries are in === + // === the same order. If not, the atomic boundaries are swaped to the === + // === correct order. === + + for (RankToBoundMap::iterator rankIt = other.begin(); + rankIt != other.end(); ++rankIt) { + + // === We have received from rank "rankIt->first" the ordered list of === + // === element indices. Now, we have to sort the corresponding list in === + // === this rank to get the same order. === + + for (unsigned int j = 0; j < rankIt->second.size(); j++) { + + // If the expected object is not at place, search for it. + + BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj; + + if ((rankIt->second)[j].neighObj != recvedBound) { + unsigned int k = j + 1; + + for (; k < rankIt->second.size(); k++) + if ((rankIt->second)[k].neighObj == recvedBound) + break; + + // The element must always be found, because the list is just in + // another order. + TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); + + // Swap the current with the found element. + AtomicBoundary tmpBound = (rankIt->second)[k]; + (rankIt->second)[k] = (rankIt->second)[j]; + (rankIt->second)[j] = tmpBound; + } + } + } + + + // === Do the same for the periodic boundaries. === + + if (periodic.size() > 0) { + stdMpi.clear(); + + RankToBoundMap sendBounds, recvBounds; + for (RankToBoundMap::iterator rankIt = periodic.begin(); + rankIt != periodic.end(); ++rankIt) { + + if (rankIt->first == mpiRank) + continue; + + if (rankIt->first < mpiRank) + sendBounds[rankIt->first] = rankIt->second; + else + recvBounds[rankIt->first] = rankIt->second; + } - boundary.clear(); + stdMpi.send(sendBounds); + stdMpi.recv(recvBounds); + stdMpi.startCommunication(); + + for (RankToBoundMap::iterator rankIt = periodic.begin(); + rankIt != periodic.end(); ++rankIt) { + + if (rankIt->first <= mpiRank) + continue; + + for (unsigned int j = 0; j < rankIt->second.size(); j++) { + BoundaryObject &recvRankObj = + stdMpi.getRecvData()[rankIt->first][j].rankObj; + BoundaryObject &recvNeighObj = + stdMpi.getRecvData()[rankIt->first][j].neighObj; + + if (periodic[rankIt->first][j].neighObj != recvRankObj || + periodic[rankIt->first][j].rankObj != recvNeighObj) { + unsigned int k = j + 1; + for (; k < rankIt->second.size(); k++) + if (periodic[rankIt->first][k].neighObj == recvRankObj && + periodic[rankIt->first][k].rankObj == recvNeighObj) + break; + + // The element must always be found, because the list is just in + // another order. + TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); + + // Swap the current with the found element. + AtomicBoundary tmpBound = (rankIt->second)[k]; + (rankIt->second)[k] = (rankIt->second)[j]; + (rankIt->second)[j] = tmpBound; + } + } + } + } // periodicBoundary.boundary.size() > 0 } diff --git a/AMDiS/src/parallel/InteriorBoundary.h b/AMDiS/src/parallel/InteriorBoundary.h index ab830e41..2048e1f1 100644 --- a/AMDiS/src/parallel/InteriorBoundary.h +++ b/AMDiS/src/parallel/InteriorBoundary.h @@ -29,6 +29,7 @@ #include "AMDiS_fwd.h" #include "BoundaryObject.h" #include "parallel/MeshLevelData.h" +#include "parallel/ParallelTypes.h" namespace AMDiS { @@ -40,12 +41,35 @@ namespace AMDiS { */ class InteriorBoundary { public: - typedef map > RankToBoundMap; + void create(MPI::Intracomm &mpiComm, + ElementObjectDatabase &elObjDb); + AtomicBoundary& getNewAtomicOwn(int rank); + + AtomicBoundary& getNewAtomicOther(int rank); + + AtomicBoundary& getNewAtomicPer(int rank); + + /// Writes this object to a file. + void serialize(ostream &out); + + /// Reads the state of an interior boundary from a file. + void deserialize(istream &in, map &elIndexMap); + + private: + void serializeExcludeList(ostream &out, ExcludeList &list); + + void deserializeExcludeList(istream &in, ExcludeList &list); + + public: + RankToBoundMap own, other, periodic; + + + public: /// Iterator for the interior boundary object. class iterator { public: - iterator(InteriorBoundary &b, int traverseLevel = 0) + iterator(RankToBoundMap &b, int traverseLevel = 0) : bound(b), level(traverseLevel) { @@ -55,14 +79,14 @@ namespace AMDiS { /// Set the iterator to the first position. void reset() { - mapIt = bound.boundary.begin(); + mapIt = bound.begin(); nextNonempty(); } /// Test if iterator is at the final position. bool end() const { - return (mapIt == bound.boundary.end()); + return (mapIt == bound.end()); } /// Move iterator to the next position. @@ -105,13 +129,13 @@ namespace AMDiS { { do { // Return, we are at the end. - if (mapIt == bound.boundary.end()) + if (mapIt == bound.end()) return; // Search for the next non empty boundary map. while (mapIt->second.size() == 0) { ++mapIt; - if (mapIt == bound.boundary.end()) + if (mapIt == bound.end()) return; } @@ -137,33 +161,10 @@ namespace AMDiS { vector::iterator vecIt; - InteriorBoundary &bound; + RankToBoundMap &bound; int level; }; - - public: - void clear(); - - AtomicBoundary& getNewAtomic(int rank); - - /// Writes this object to a file. - void serialize(ostream &out); - - /// Reads the state of an interior boundary from a file. - void deserialize(istream &in, map &elIndexMap); - - /// Compares this interior boundaries with some other. The order of the - /// boundary elements within the object does not play a role. - bool operator==(const InteriorBoundary& other) const; - - protected: - void serializeExcludeList(ostream &out, ExcludeList &list); - - void deserializeExcludeList(istream &in, ExcludeList &list); - - public: - RankToBoundMap boundary; }; } diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc index 46df2f1b..691d622d 100644 --- a/AMDiS/src/parallel/MeshDistributor.cc +++ b/AMDiS/src/parallel/MeshDistributor.cc @@ -87,7 +87,6 @@ namespace AMDiS { debugOutputDir(""), lastMeshChangeIndex(0), createBoundaryDofFlag(0), - sebastianMode(false), boundaryDofInfo(1) { FUNCNAME("MeshDistributor::ParalleDomainBase()"); @@ -160,7 +159,7 @@ namespace AMDiS { } } while (doNext); - elObjDb.setMesh(feSpaces[0]->getMesh()); + elObjDb.setFeSpace(feSpaces[0]); // If the problem has been already read from a file, we need only to set // isRankDofs to all matrices and rhs vector and to remove periodic @@ -172,14 +171,7 @@ namespace AMDiS { removePeriodicBoundaryConditions(); - macroElIndexMap.clear(); - macroElIndexTypeMap.clear(); - - for (vector::iterator it = allMacroElements.begin(); - it != allMacroElements.end(); ++it) { - macroElIndexMap.insert(make_pair((*it)->getIndex(), (*it)->getElement())); - macroElIndexTypeMap.insert(make_pair((*it)->getIndex(), (*it)->getElType())); - } + elObjDb.createMacroElementInfo(allMacroElements); createBoundaryDofs(); @@ -629,9 +621,9 @@ namespace AMDiS { for (std::set::iterator elIt0 = el0.begin(); elIt0 != el0.end(); ++elIt0) { for (std::set::iterator elIt1 = el1.begin(); elIt1 != el1.end(); ++elIt1) { pair edge0 = - make_pair(macroElIndexMap[*elIt0], edgeNoInEl[*elIt0]); + make_pair(elObjDb.getElementPtr(*elIt0), edgeNoInEl[*elIt0]); pair edge1 = - make_pair(macroElIndexMap[*elIt1], edgeNoInEl[*elIt1]); + make_pair(elObjDb.getElementPtr(*elIt1), edgeNoInEl[*elIt1]); DofEdge dofEdge0 = edge0.first->getEdge(edge0.second); DofEdge dofEdge1 = edge1.first->getEdge(edge1.second); @@ -915,17 +907,17 @@ namespace AMDiS { // important. Therefore, we add all boundaries to one boundary container. RankToBoundMap allBound; - for (InteriorBoundary::iterator it(rankIntBoundary); !it.end(); ++it) + for (InteriorBoundary::iterator it(intBoundary.own); !it.end(); ++it) if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) || (mesh->getDim() == 3 && it->rankObj.subObj == FACE)) allBound[it.getRank()].push_back(*it); - for (InteriorBoundary::iterator it(otherIntBoundary); !it.end(); ++it) + for (InteriorBoundary::iterator it(intBoundary.other); !it.end(); ++it) if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) || (mesh->getDim() == 3 && it->rankObj.subObj == FACE)) allBound[it.getRank()].push_back(*it); - for (InteriorBoundary::iterator it(periodicBoundary); !it.end(); ++it) { + for (InteriorBoundary::iterator it(intBoundary.periodic); !it.end(); ++it) { if (it.getRank() == mpiRank) { if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) || (mesh->getDim() == 3 && it->rankObj.subObj == FACE)) { @@ -1516,8 +1508,9 @@ namespace AMDiS { { FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()"); - createMeshElementData(); - createBoundaryData(); + elObjDb.create(); + elObjDb.createRankData(partitionMap, levelData); + intBoundary.create(mpiComm, elObjDb); } @@ -1526,7 +1519,7 @@ namespace AMDiS { FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()"); elObjDb.createRankData(partitionMap, levelData); - createBoundaryData(); + intBoundary.create(mpiComm, elObjDb); #if (DEBUG != 0) ParallelDebug::printBoundaryInfo(*this); @@ -1534,377 +1527,6 @@ namespace AMDiS { } - void MeshDistributor::createMeshElementData() - { - FUNCNAME("MeshDistributor::createMeshElementData()"); - - - // === Fills macro element data structures. === - - TraverseStack stack; - ElInfo *elInfo = stack.traverseFirst(mesh, -1, - Mesh::CALL_LEAF_EL | - Mesh::FILL_NEIGH | - Mesh::FILL_BOUND); - while (elInfo) { - TEST_EXIT_DBG(elInfo->getLevel() == 0)("Should not happen!\n"); - - Element *el = elInfo->getElement(); - macroElIndexMap.insert(make_pair(el->getIndex(), el)); - macroElIndexTypeMap.insert(make_pair(el->getIndex(), elInfo->getType())); - - // Add all sub object of the element to the variable elObjDb. - elObjDb.addElement(elInfo); - - elInfo = stack.traverseNext(elInfo); - } - - - // Create periodic data, if there are periodic boundary conditions. - elObjDb.createPeriodicData(feSpaces[0]); - - // Create data about the reverse modes of neighbouring elements. - elObjDb.createReverseModeData(feSpaces[0], macroElIndexMap, - macroElIndexTypeMap); - - // Create mesh element data for this rank. - elObjDb.createRankData(partitionMap, levelData); - } - - - void MeshDistributor::createBoundaryData() - { - FUNCNAME("MeshDistributor::createBoundaryData()"); - - // === Clear all relevant data structures. === - - rankIntBoundary.clear(); - otherIntBoundary.clear(); - periodicBoundary.clear(); - - // === Create interior boundary data structure. === - - for (int geoPos = 0; geoPos < mesh->getDim(); geoPos++) { - GeoIndex geoIndex = INDEX_OF_DIM(geoPos, mesh->getDim()); - - while (elObjDb.iterate(geoIndex)) { - map& objData = elObjDb.getIterateData(); - if (!(objData.count(mpiRank) && objData.size() > 1)) - continue; - - int owner = elObjDb.getIterateOwner(); - ElementObjectData& rankBoundEl = objData[mpiRank]; - - AtomicBoundary bound; - bound.maxLevel = elObjDb.getIterateMaxLevel(); - - bound.rankObj.el = macroElIndexMap[rankBoundEl.elIndex]; - bound.rankObj.elIndex = rankBoundEl.elIndex; - bound.rankObj.elType = macroElIndexTypeMap[rankBoundEl.elIndex]; - bound.rankObj.subObj = geoIndex; - bound.rankObj.ithObj = rankBoundEl.ithObject; - - if (geoIndex == FACE) { - for (int edgeNo = 0; edgeNo < 3; edgeNo++) { - int edgeOfFace = - bound.rankObj.el->getEdgeOfFace(bound.rankObj.ithObj, edgeNo); - - bound.rankObj.excludedSubstructures.push_back(make_pair(EDGE, edgeOfFace)); - } - } - - - if (owner == mpiRank) { - for (map::iterator it2 = objData.begin(); - it2 != objData.end(); ++it2) { - if (it2->first == mpiRank) - continue; - - bound.neighObj.el = macroElIndexMap[it2->second.elIndex]; - bound.neighObj.elIndex = it2->second.elIndex; - bound.neighObj.elType = macroElIndexTypeMap[it2->second.elIndex]; - bound.neighObj.subObj = geoIndex; - bound.neighObj.ithObj = it2->second.ithObject; - - bound.type = INTERIOR; - - AtomicBoundary& b = rankIntBoundary.getNewAtomic(it2->first); - b = bound; - if (geoIndex == EDGE) - b.neighObj.reverseMode = - elObjDb.getEdgeReverseMode(rankBoundEl, it2->second); - if (geoIndex == FACE) - b.neighObj.reverseMode = - elObjDb.getFaceReverseMode(rankBoundEl, it2->second); - } - - } else { - TEST_EXIT_DBG(objData.count(owner) == 1) - ("Should not happen!\n"); - - ElementObjectData& ownerBoundEl = objData[owner]; - - bound.neighObj.el = macroElIndexMap[ownerBoundEl.elIndex]; - bound.neighObj.elIndex = ownerBoundEl.elIndex; - bound.neighObj.elType = -1; - bound.neighObj.subObj = geoIndex; - bound.neighObj.ithObj = ownerBoundEl.ithObject; - - bound.type = INTERIOR; - - AtomicBoundary& b = otherIntBoundary.getNewAtomic(owner); - b = bound; - if (geoIndex == EDGE) - b.rankObj.reverseMode = - elObjDb.getEdgeReverseMode(rankBoundEl, ownerBoundEl); - if (geoIndex == FACE) - b.rankObj.reverseMode = - elObjDb.getFaceReverseMode(rankBoundEl, ownerBoundEl); - } - } - } - - - // === Create periodic boundary data structure. === - - for (PerBoundMap::iterator it = elObjDb.getPeriodicVertices().begin(); - it != elObjDb.getPeriodicVertices().end(); ++it) { - if (elObjDb.isInRank(it->first.first, mpiRank) == false) - continue; - - ElementObjectData& perDofEl0 = - elObjDb.getElementsInRank(it->first.first)[mpiRank]; - - for (map::iterator elIt = elObjDb.getElementsInRank(it->first.second).begin(); - elIt != elObjDb.getElementsInRank(it->first.second).end(); ++elIt) { - - int otherElementRank = elIt->first; - ElementObjectData& perDofEl1 = elIt->second; - - AtomicBoundary bound; - bound.rankObj.el = macroElIndexMap[perDofEl0.elIndex]; - bound.rankObj.elIndex = perDofEl0.elIndex; - bound.rankObj.elType = macroElIndexTypeMap[perDofEl0.elIndex]; - bound.rankObj.subObj = VERTEX; - bound.rankObj.ithObj = perDofEl0.ithObject; - - bound.neighObj.el = macroElIndexMap[perDofEl1.elIndex]; - bound.neighObj.elIndex = perDofEl1.elIndex; - bound.neighObj.elType = macroElIndexTypeMap[perDofEl1.elIndex]; - bound.neighObj.subObj = VERTEX; - bound.neighObj.ithObj = perDofEl1.ithObject; - - bound.type = it->second; - - if (sebastianMode) { - if (bound.rankObj.elIndex == 3 && - bound.rankObj.ithObj == 1 || - bound.rankObj.elIndex == 78 && - bound.rankObj.ithObj == 2) { - AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank); - b = bound; - } - } else { - AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank); - b = bound; - } - } - } - - - for (PerBoundMap::iterator it = elObjDb.getPeriodicEdges().begin(); - it != elObjDb.getPeriodicEdges().end(); ++it) { - if (elObjDb.isInRank(it->first.first, mpiRank) == false) - continue; - - ElementObjectData& perEdgeEl0 = elObjDb.getElementsInRank(it->first.first)[mpiRank]; - - for (map::iterator elIt = elObjDb.getElementsInRank(it->first.second).begin(); - elIt != elObjDb.getElementsInRank(it->first.second).end(); ++elIt) { - - int otherElementRank = elIt->first; - ElementObjectData& perEdgeEl1 = elIt->second; - - AtomicBoundary bound; - bound.rankObj.el = macroElIndexMap[perEdgeEl0.elIndex]; - bound.rankObj.elIndex = perEdgeEl0.elIndex; - bound.rankObj.elType = macroElIndexTypeMap[perEdgeEl0.elIndex]; - bound.rankObj.subObj = EDGE; - bound.rankObj.ithObj = perEdgeEl0.ithObject; - - bound.neighObj.el = macroElIndexMap[perEdgeEl1.elIndex]; - bound.neighObj.elIndex = perEdgeEl1.elIndex; - bound.neighObj.elType = macroElIndexTypeMap[perEdgeEl1.elIndex]; - bound.neighObj.subObj = EDGE; - bound.neighObj.ithObj = perEdgeEl1.ithObject; - - bound.type = it->second; - - AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank); - b = bound; - - if (mpiRank > otherElementRank) - b.neighObj.reverseMode = - elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); - else - b.rankObj.reverseMode = - elObjDb.getEdgeReverseMode(perEdgeEl0, perEdgeEl1); - } - } - - - for (PerBoundMap::iterator it = elObjDb.getPeriodicFaces().begin(); - it != elObjDb.getPeriodicFaces().end(); ++it) { - if (elObjDb.isInRank(it->first.first, mpiRank) == false) - continue; - - TEST_EXIT_DBG(elObjDb.getElements(it->first.first).size() == 1) - ("Should not happen!\n"); - TEST_EXIT_DBG(elObjDb.getElements(it->first.second).size() == 1) - ("Should not happen!\n"); - - ElementObjectData& perFaceEl0 = elObjDb.getElementsInRank(it->first.first)[mpiRank]; - - for (map::iterator elIt = elObjDb.getElementsInRank(it->first.second).begin(); - elIt != elObjDb.getElementsInRank(it->first.second).end(); ++elIt) { - - int otherElementRank = elIt->first; - ElementObjectData& perFaceEl1 = elIt->second; - - AtomicBoundary bound; - bound.rankObj.el = macroElIndexMap[perFaceEl0.elIndex]; - bound.rankObj.elIndex = perFaceEl0.elIndex; - bound.rankObj.elType = macroElIndexTypeMap[perFaceEl0.elIndex]; - bound.rankObj.subObj = FACE; - bound.rankObj.ithObj = perFaceEl0.ithObject; - - bound.neighObj.el = macroElIndexMap[perFaceEl1.elIndex]; - bound.neighObj.elIndex = perFaceEl1.elIndex; - bound.neighObj.elType = macroElIndexTypeMap[perFaceEl1.elIndex]; - bound.neighObj.subObj = FACE; - bound.neighObj.ithObj = perFaceEl1.ithObject; - - bound.type = it->second; - - AtomicBoundary& b = periodicBoundary.getNewAtomic(otherElementRank); - b = bound; - - if (mpiRank > otherElementRank) - b.neighObj.reverseMode = - elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1); - else - b.rankObj.reverseMode = - elObjDb.getFaceReverseMode(perFaceEl0, perFaceEl1); - } - } - - - // === Once we have this information, we must care about the order of the === - // === atomic bounds in the three boundary handling object. Eventually === - // === all the boundaries have to be in the same order on both ranks that === - // === share the bounday. === - - StdMpi > stdMpi(mpiComm); - stdMpi.send(rankIntBoundary.boundary); - stdMpi.recv(otherIntBoundary.boundary); - stdMpi.startCommunication(); - - - // === The information about all neighbouring boundaries has been === - // === received. So the rank tests if its own atomic boundaries are in === - // === the same order. If not, the atomic boundaries are swaped to the === - // === correct order. === - - for (RankToBoundMap::iterator rankIt = otherIntBoundary.boundary.begin(); - rankIt != otherIntBoundary.boundary.end(); ++rankIt) { - - // === We have received from rank "rankIt->first" the ordered list of === - // === element indices. Now, we have to sort the corresponding list in === - // === this rank to get the same order. === - - for (unsigned int j = 0; j < rankIt->second.size(); j++) { - - // If the expected object is not at place, search for it. - - BoundaryObject &recvedBound = stdMpi.getRecvData()[rankIt->first][j].rankObj; - - if ((rankIt->second)[j].neighObj != recvedBound) { - unsigned int k = j + 1; - - for (; k < rankIt->second.size(); k++) - if ((rankIt->second)[k].neighObj == recvedBound) - break; - - // The element must always be found, because the list is just in - // another order. - TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); - - // Swap the current with the found element. - AtomicBoundary tmpBound = (rankIt->second)[k]; - (rankIt->second)[k] = (rankIt->second)[j]; - (rankIt->second)[j] = tmpBound; - } - } - } - - - // === Do the same for the periodic boundaries. === - - if (periodicBoundary.boundary.size() > 0) { - stdMpi.clear(); - - InteriorBoundary sendBounds, recvBounds; - for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin(); - rankIt != periodicBoundary.boundary.end(); ++rankIt) { - - if (rankIt->first == mpiRank) - continue; - - if (rankIt->first < mpiRank) - sendBounds.boundary[rankIt->first] = rankIt->second; - else - recvBounds.boundary[rankIt->first] = rankIt->second; - } - - stdMpi.send(sendBounds.boundary); - stdMpi.recv(recvBounds.boundary); - stdMpi.startCommunication(); - - for (RankToBoundMap::iterator rankIt = periodicBoundary.boundary.begin(); - rankIt != periodicBoundary.boundary.end(); ++rankIt) { - - if (rankIt->first <= mpiRank) - continue; - - for (unsigned int j = 0; j < rankIt->second.size(); j++) { - BoundaryObject &recvRankObj = - stdMpi.getRecvData()[rankIt->first][j].rankObj; - BoundaryObject &recvNeighObj = - stdMpi.getRecvData()[rankIt->first][j].neighObj; - - if (periodicBoundary.boundary[rankIt->first][j].neighObj != recvRankObj || - periodicBoundary.boundary[rankIt->first][j].rankObj != recvNeighObj) { - unsigned int k = j + 1; - for (; k < rankIt->second.size(); k++) - if (periodicBoundary.boundary[rankIt->first][k].neighObj == recvRankObj && - periodicBoundary.boundary[rankIt->first][k].rankObj == recvNeighObj) - break; - - // The element must always be found, because the list is just in - // another order. - TEST_EXIT_DBG(k < rankIt->second.size())("Should never happen!\n"); - - // Swap the current with the found element. - AtomicBoundary tmpBound = (rankIt->second)[k]; - (rankIt->second)[k] = (rankIt->second)[j]; - (rankIt->second)[j] = tmpBound; - } - } - } - } // periodicBoundary.boundary.size() > 0 - } - - void MeshDistributor::createBoundaryDofs() { FUNCNAME("MeshDistributor::createBoundaryDofs()"); @@ -1912,84 +1534,68 @@ namespace AMDiS { int nLevels = levelData.getLevelNumber(); TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); - sendDofs.init(nLevels); - recvDofs.init(nLevels); + sendDofs.init(nLevels, feSpaces); + sendDofs.create(intBoundary.own); - boundaryDofInfo.resize(nLevels); + recvDofs.init(nLevels, feSpaces); + recvDofs.create(intBoundary.other); - for (unsigned int i = 0; i < feSpaces.size(); i++) - for (int j = 0; j < nLevels; j++) - createBoundaryDofs(feSpaces[i], j); + createBoundaryDofInfo(); } - void MeshDistributor::createBoundaryDofs(const FiniteElemSpace *feSpace, - int level) + void MeshDistributor::createBoundaryDofInfo() { - FUNCNAME("MeshDistributor::createBoundaryDofs()"); + FUNCNAME("MeshDistributor::createBoundaryDofInfo()"); - if (createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) { - - TEST_EXIT(level < boundaryDofInfo.size())("Should not happen!\n"); + if (!createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) + return; - // === Clear data. === - for (int geo = FACE; geo >= VERTEX; geo--) - boundaryDofInfo[level][feSpace].geoDofs[static_cast(geo)].clear(); + int nLevels = levelData.getLevelNumber(); + boundaryDofInfo.resize(nLevels); - // === Create send DOFs. === - for (int geo = FACE; geo >= VERTEX; geo--) { - for (InteriorBoundary::iterator it(rankIntBoundary, level); - !it.end(); ++it) { - if (it->rankObj.subObj == geo) { - DofContainer dofs; - it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); + for (unsigned int i = 0; i < feSpaces.size(); i++) { + const FiniteElemSpace *feSpace = feSpaces[i]; - DofContainer& tmp = - sendDofs.getDofContainer(it.getRank(), feSpace, level); - tmp.insert(tmp.end(), dofs.begin(), dofs.end()); + for (int level = 0; level < nLevels; level++) { - if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS)) - boundaryDofInfo[level][feSpace].geoDofs[static_cast(geo)].insert(dofs.begin(), dofs.end()); + // === Clear data. === + for (int geo = FACE; geo >= VERTEX; geo--) + boundaryDofInfo[level][feSpace].geoDofs[static_cast(geo)].clear(); + + // === Create send DOFs. === + for (int geo = FACE; geo >= VERTEX; geo--) { + for (InteriorBoundary::iterator it(intBoundary.own, level); + !it.end(); ++it) { + if (it->rankObj.subObj == geo) { + DofContainer dofs; + it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); + + if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS)) + boundaryDofInfo[level][feSpace]. + geoDofs[static_cast(geo)].insert(dofs.begin(), dofs.end()); + } } } - } - - // === Create recv DOFs. === - for (int geo = FACE; geo >= VERTEX; geo--) { - for (InteriorBoundary::iterator it(otherIntBoundary, level); - !it.end(); ++it) { - if (it->rankObj.subObj == geo) { - DofContainer dofs; - it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); - - DofContainer& tmp = - recvDofs.getDofContainer(it.getRank(), feSpace, level); - tmp.insert(tmp.end(), dofs.begin(), dofs.end()); - - if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS)) - boundaryDofInfo[level][feSpace].geoDofs[static_cast(geo)].insert(dofs.begin(), dofs.end()); + + // === Create recv DOFs. === + for (int geo = FACE; geo >= VERTEX; geo--) { + for (InteriorBoundary::iterator it(intBoundary.other, level); + !it.end(); ++it) { + if (it->rankObj.subObj == geo) { + DofContainer dofs; + it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); + + if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS)) + boundaryDofInfo[level][feSpace]. + geoDofs[static_cast(geo)].insert(dofs.begin(), dofs.end()); + } } } } - } else { - for (InteriorBoundary::iterator it(rankIntBoundary, level); - !it.end(); ++it) - it->rankObj.el->getAllDofs(feSpace, it->rankObj, - sendDofs.getDofContainer(it.getRank(), feSpace, level)); - - for (InteriorBoundary::iterator it(otherIntBoundary, level); - !it.end(); ++it) - it->rankObj.el->getAllDofs(feSpace, it->rankObj, - recvDofs.getDofContainer(it.getRank(), feSpace, level)); } - - // === Delete all empty DOF send and recv positions === - - sendDofs.removeEmpty(); - recvDofs.removeEmpty(); } - void MeshDistributor::removeMacroElements() { FUNCNAME("MeshDistributor::removeMacroElements()"); @@ -2106,16 +1712,19 @@ namespace AMDiS { FUNCNAME("MeshDistributor::createPeriodicMap()"); // Clear all periodic DOF mappings calculated before. We do it from scratch. - periodicDofs.init(levelData.getLevelNumber()); + periodicDofs.init(levelData.getLevelNumber(), feSpaces); periodicMap.clear(); // If there are no periodic boundaries, return. Note that periodicDofs and // periodicMap must be still cleared before: if we do repartitioning and // there were periodic boundaries in subdomain before and after repartitioning // there are no more periodic boundaries. - if (periodicBoundary.boundary.size() == 0) + if (intBoundary.periodic.size() == 0) return; + TEST_EXIT(levelData.getLevelNumber() == 1) + ("Periodic DOF map does not support multi level domain decomposition!\n"); + for (unsigned int i = 0; i < feSpaces.size(); i++) createPeriodicMap(feSpaces[i]); } @@ -2132,8 +1741,8 @@ namespace AMDiS { map > rankToDofType; - for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin(); - it != periodicBoundary.boundary.end(); ++it) { + for (RankToBoundMap::iterator it = intBoundary.periodic.begin(); + it != intBoundary.periodic.end(); ++it) { if (it->first == mpiRank) { // Here we have a periodic boundary within rank's subdomain. So we can @@ -2201,8 +1810,8 @@ namespace AMDiS { // === DOFs from the other ranks. === - for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin(); - it != periodicBoundary.boundary.end(); ++it) { + for (RankToBoundMap::iterator it = intBoundary.periodic.begin(); + it != intBoundary.periodic.end(); ++it) { DofContainer& dofs = periodicDofs.getDofContainer(it->first, feSpace); vector& types = rankToDofType[it->first]; @@ -2225,8 +1834,8 @@ namespace AMDiS { StdMpi stdMpi2(mpiComm); - for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin(); - it != periodicBoundary.boundary.end(); ++it) { + for (RankToBoundMap::iterator it = intBoundary.periodic.begin(); + it != intBoundary.periodic.end(); ++it) { if (it->first == mpiRank) continue; @@ -2321,9 +1930,7 @@ namespace AMDiS { elObjDb.serialize(out); - rankIntBoundary.serialize(out); - otherIntBoundary.serialize(out); - periodicBoundary.serialize(out); + intBoundary.serialize(out); serialize(out, sendDofs.getData()); serialize(out, recvDofs.getData()); @@ -2382,9 +1989,7 @@ namespace AMDiS { elObjDb.deserialize(in); - rankIntBoundary.deserialize(in, elIndexMap); - otherIntBoundary.deserialize(in, elIndexMap); - periodicBoundary.deserialize(in, elIndexMap); + intBoundary.deserialize(in, elIndexMap); deserialize(in, sendDofs.getData(), dofIndexMap); deserialize(in, recvDofs.getData(), dofIndexMap); diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h index 7aeb3037..ba580061 100644 --- a/AMDiS/src/parallel/MeshDistributor.h +++ b/AMDiS/src/parallel/MeshDistributor.h @@ -293,13 +293,9 @@ namespace AMDiS { void updateInteriorBoundaryInfo(); - void createMeshElementData(); - - void createBoundaryData(); - void createBoundaryDofs(); - void createBoundaryDofs(const FiniteElemSpace *feSpace, int level); + void createBoundaryDofInfo(); /// Removes all macro elements from the mesh that are not part of ranks /// partition. @@ -315,21 +311,17 @@ namespace AMDiS { /// handled by the mesh distributor. void createPeriodicMap(); - /** \brief - * Creates, for a specific FE space, to all DOFs in rank's partition that - * are on a periodic boundary the mapping from dof index to the other - * periodic dof indices. This information is stored in \ref periodicDofMap. - */ + /// Creates, for a specific FE space, to all DOFs in rank's partition that + /// are on a periodic boundary the mapping from dof index to the other + /// periodic dof indices. This information is stored in \ref periodicDofMap. void createPeriodicMap(const FiniteElemSpace *feSpace); - /** \brief - * This function is called only once during the initialization when the - * whole macro mesh is available on all cores. It copies the pointers of all - * macro elements to \ref allMacroElements and stores all neighbour - * information based on macro element indices (and not pointer based) in - * \ref macroElementNeighbours. These information are then used to - * reconstruct macro elements during mesh redistribution. - */ + /// This function is called only once during the initialization when the + /// whole macro mesh is available on all cores. It copies the pointers of all + /// macro elements to \ref allMacroElements and stores all neighbour + /// information based on macro element indices (and not pointer based) in + /// \ref macroElementNeighbours. These information are then used to + /// reconstruct macro elements during mesh redistribution. void createMacroElementInfo(); void updateMacroElementInfo(); @@ -481,27 +473,9 @@ namespace AMDiS { /// macro mesh. ElementObjectDatabase elObjDb; - /// Maps to each macro element index a pointer to the corresponding element. - map macroElIndexMap; - - /// Maps to each macro element index the type of this element. - map macroElIndexTypeMap; - - /// Defines the interior boundaries of the domain that result from - /// partitioning the whole mesh. Contains only the boundaries, which are - /// owned by the rank, i.e., the object gives for every neighbour rank i - /// the boundaries this rank owns and shares with rank i. - InteriorBoundary rankIntBoundary; - /// Defines the interior boundaries of the domain that result from - /// partitioning the whole mesh. Contains only the boundaries, which are - /// not owned by the rank, i.e., the object gives for every neighbour rank - /// i the boundaries that are owned by rank i and are shared with this rank. - InteriorBoundary otherIntBoundary; - - /// Defines the periodic boundaries with other ranks. Periodic boundaries - /// have no owner, as it is the case of interior boundaries. - InteriorBoundary periodicBoundary; + /// partitioning the whole mesh. + InteriorBoundary intBoundary; /// This map contains for each rank the list of DOFs the current rank must /// end to exchange solution DOFs at the interior boundaries. @@ -574,8 +548,6 @@ namespace AMDiS { MeshLevelData levelData; public: - bool sebastianMode; - /// The boundary DOFs are sorted by subobject entities, i.e., first all /// face DOFs, edge DOFs and to the last vertex DOFs will be set to /// communication structure vectors, \ref sendDofs and \ref recvDofs. diff --git a/AMDiS/src/parallel/ParallelDebug.cc b/AMDiS/src/parallel/ParallelDebug.cc index 2cef40f5..a9cab1df 100644 --- a/AMDiS/src/parallel/ParallelDebug.cc +++ b/AMDiS/src/parallel/ParallelDebug.cc @@ -31,16 +31,16 @@ namespace AMDiS { vector sendBuffers, recvBuffers; - MPI::Request request[pdb.rankIntBoundary.boundary.size() + - pdb.otherIntBoundary.boundary.size() + - pdb.periodicBoundary.boundary.size() * 2]; + MPI::Request request[pdb.intBoundary.own.size() + + pdb.intBoundary.other.size() + + pdb.intBoundary.periodic.size() * 2]; int requestCounter = 0; // === Send rank's boundary information. === - for (RankToBoundMap::iterator rankIt = pdb.rankIntBoundary.boundary.begin(); - rankIt != pdb.rankIntBoundary.boundary.end(); ++rankIt) { + for (RankToBoundMap::iterator rankIt = pdb.intBoundary.own.begin(); + rankIt != pdb.intBoundary.own.end(); ++rankIt) { int nSendInt = rankIt->second.size(); int* buffer = new int[nSendInt]; @@ -56,8 +56,8 @@ namespace AMDiS { // === Receive information from other ranks about the interior boundaries. ==== - for (RankToBoundMap::iterator rankIt = pdb.otherIntBoundary.boundary.begin(); - rankIt != pdb.otherIntBoundary.boundary.end(); ++rankIt) { + for (RankToBoundMap::iterator rankIt = pdb.intBoundary.other.begin(); + rankIt != pdb.intBoundary.other.end(); ++rankIt) { int nRecvInt = rankIt->second.size(); int *buffer = new int[nRecvInt]; recvBuffers.push_back(buffer); @@ -69,8 +69,8 @@ namespace AMDiS { // === To the last, do the same of periodic boundaries. === - for (RankToBoundMap::iterator rankIt = pdb.periodicBoundary.boundary.begin(); - rankIt != pdb.periodicBoundary.boundary.end(); ++rankIt) { + for (RankToBoundMap::iterator rankIt = pdb.intBoundary.periodic.begin(); + rankIt != pdb.intBoundary.periodic.end(); ++rankIt) { if (rankIt->first == pdb.mpiRank) continue; @@ -103,16 +103,16 @@ namespace AMDiS { // === and after this the periodic ones. === int bufCounter = 0; - for (RankToBoundMap::iterator rankIt = pdb.otherIntBoundary.boundary.begin(); - rankIt != pdb.otherIntBoundary.boundary.end(); ++rankIt) { + for (RankToBoundMap::iterator rankIt = pdb.intBoundary.other.begin(); + rankIt != pdb.intBoundary.other.end(); ++rankIt) { TEST_EXIT(rankIt->second.size() == - pdb.otherIntBoundary.boundary[rankIt->first].size()) + pdb.intBoundary.other[rankIt->first].size()) ("Boundaries does not fit together!\n"); for (unsigned int i = 0; i < rankIt->second.size(); i++) { int elIndex1 = recvBuffers[bufCounter][i]; - int elIndex2 = pdb.otherIntBoundary.boundary[rankIt->first][i].neighObj.elIndex; + int elIndex2 = pdb.intBoundary.other[rankIt->first][i].neighObj.elIndex; TEST_EXIT(elIndex1 == elIndex2)("Wrong element index at interior boundary!\n"); } @@ -121,18 +121,18 @@ namespace AMDiS { } - for (RankToBoundMap::iterator rankIt = pdb.periodicBoundary.boundary.begin(); - rankIt != pdb.periodicBoundary.boundary.end(); ++rankIt) { + for (RankToBoundMap::iterator rankIt = pdb.intBoundary.periodic.begin(); + rankIt != pdb.intBoundary.periodic.end(); ++rankIt) { if (rankIt->first == pdb.mpiRank) continue; for (unsigned int i = 0; i < rankIt->second.size(); i++) { int elIndex1 = recvBuffers[bufCounter][i]; - int elIndex2 = pdb.periodicBoundary.boundary[rankIt->first][i].neighObj.elIndex; + int elIndex2 = pdb.intBoundary.periodic[rankIt->first][i].neighObj.elIndex; TEST_EXIT(elIndex1 == elIndex2) ("Wrong element index at periodic boundary el %d with rank %d: %d %d\n", - pdb.periodicBoundary.boundary[rankIt->first][i].rankObj.elIndex, + pdb.intBoundary.periodic[rankIt->first][i].rankObj.elIndex, rankIt->first, elIndex1, elIndex2); } @@ -256,8 +256,8 @@ namespace AMDiS { RankToCoords sendCoords; map > rankToDofType; - for (InteriorBoundary::RankToBoundMap::iterator it = pdb.periodicBoundary.boundary.begin(); - it != pdb.periodicBoundary.boundary.end(); ++it) { + for (RankToBoundMap::iterator it = pdb.intBoundary.periodic.begin(); + it != pdb.intBoundary.periodic.end(); ++it) { if (it->first == pdb.mpiRank) continue; @@ -717,7 +717,7 @@ namespace AMDiS { if (tmp <= 0) return; - for (InteriorBoundary::iterator it(pdb.rankIntBoundary); !it.end(); ++it) { + for (InteriorBoundary::iterator it(pdb.intBoundary.own); !it.end(); ++it) { MSG("Rank owned boundary with rank %d: \n", it.getRank()); MSG(" ranks obj-ind: %d sub-obj: %d ith-obj: %d\n", it->rankObj.elIndex, it->rankObj.subObj, it->rankObj.ithObj); @@ -725,7 +725,7 @@ namespace AMDiS { it->neighObj.elIndex, it->neighObj.subObj, it->neighObj.ithObj); } - for (InteriorBoundary::iterator it(pdb.otherIntBoundary); !it.end(); ++it) { + for (InteriorBoundary::iterator it(pdb.intBoundary.other); !it.end(); ++it) { MSG("Other owned boundary with rank %d: \n", it.getRank()); MSG(" ranks obj-ind: %d sub-obj: %d ith-obj: %d\n", it->rankObj.elIndex, it->rankObj.subObj, it->rankObj.ithObj); @@ -733,7 +733,7 @@ namespace AMDiS { it->neighObj.elIndex, it->neighObj.subObj, it->neighObj.ithObj); } - for (InteriorBoundary::iterator it(pdb.periodicBoundary); !it.end(); ++it) { + for (InteriorBoundary::iterator it(pdb.intBoundary.periodic); !it.end(); ++it) { MSG("Periodic boundary (ID %d) with rank %d: \n", it->type, it.getRank()); MSG(" ranks obj-ind: %d sub-obj: %d ith-obj: %d\n", @@ -862,13 +862,13 @@ namespace AMDiS { { FUNCNAME("ParallelDebug::followBoundary()"); - for (InteriorBoundary::iterator it(pdb.rankIntBoundary); !it.end(); ++it) + for (InteriorBoundary::iterator it(pdb.intBoundary.own); !it.end(); ++it) if (followThisBound(it->rankObj.elIndex, it->neighObj.elIndex)) debug::writeLocalElementDofs(pdb.mpiRank, it->rankObj.elIndex, pdb.feSpaces[0]); - for (InteriorBoundary::iterator it(pdb.otherIntBoundary); !it.end(); ++it) + for (InteriorBoundary::iterator it(pdb.intBoundary.other); !it.end(); ++it) if (followThisBound(it->rankObj.elIndex, it->neighObj.elIndex)) debug::writeLocalElementDofs(pdb.mpiRank, it->rankObj.elIndex, diff --git a/AMDiS/src/parallel/ParallelTypes.h b/AMDiS/src/parallel/ParallelTypes.h index 2e580a5b..fca05792 100644 --- a/AMDiS/src/parallel/ParallelTypes.h +++ b/AMDiS/src/parallel/ParallelTypes.h @@ -23,6 +23,7 @@ #include #include #include +#include "BoundaryObject.h" #include "Global.h" #include "parallel/InteriorBoundary.h" @@ -54,9 +55,6 @@ namespace AMDiS { /// Defines a mapping type from DOF indices to boolean values. typedef map DofIndexToBool; - /// Forward type (it maps rank numbers to the interior boundary objects). - typedef InteriorBoundary::RankToBoundMap RankToBoundMap; - typedef map DofIndexMap; /// Maps a boundary type, i.e., a boundary identifier index, to a periodic @@ -77,6 +75,8 @@ namespace AMDiS { typedef std::map DofMap; typedef vector MeshCodeVec; + + typedef map > RankToBoundMap; } #endif -- GitLab