Commit ff250c55 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Some performance issues.

parent 323d3c5c
...@@ -200,6 +200,13 @@ namespace AMDiS { ...@@ -200,6 +200,13 @@ namespace AMDiS {
const char *file, const char *file,
int line) int line)
{ {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#if (DEBUG == 0)
if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
return;
#endif
#endif
static int old_line = -1; static int old_line = -1;
if (!out) if (!out)
...@@ -229,6 +236,12 @@ namespace AMDiS { ...@@ -229,6 +236,12 @@ namespace AMDiS {
void Msg::print_warn(const char *format, ...) void Msg::print_warn(const char *format, ...)
{ {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#if (DEBUG == 0)
if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
return;
#endif
#endif
va_list arg; va_list arg;
char buff[255]; char buff[255];
......
...@@ -64,11 +64,9 @@ namespace AMDiS { ...@@ -64,11 +64,9 @@ namespace AMDiS {
/// destructor /// destructor
virtual ~Marker() {} virtual ~Marker() {}
/** \brief /// Marks element with newMark. If \ref maximumMarking is set, the element
* Marks element with newMark. If \ref maximumMarking is set, the element /// is marked only if the new mark is bigger than the old one. The return
* is marked only if the new mark is bigger than the old one. The return /// value specifies whether the element has been marked, or not.
* value specifies whether the element has been marked, or not.
*/
inline void setMark(Element *el, char newMark) inline void setMark(Element *el, char newMark)
{ {
char oldMark = el->getMark(); char oldMark = el->getMark();
...@@ -120,7 +118,7 @@ namespace AMDiS { ...@@ -120,7 +118,7 @@ namespace AMDiS {
return elMarkCoarsen; return elMarkCoarsen;
} }
// Returns \ref name of the Marker /// Returns \ref name of the Marker
inline std::string getName() const inline std::string getName() const
{ {
return name; return name;
...@@ -133,10 +131,8 @@ namespace AMDiS { ...@@ -133,10 +131,8 @@ namespace AMDiS {
/// Name of the scalar marker. /// Name of the scalar marker.
std::string name; std::string name;
/** \brief /// Equal to -1 for scalar problems. Component number if marker is
* Equal to -1 for scalar problems. Component number if marker is /// part of a vector valued marker.
* part of a vector valued marker.
*/
int row; int row;
/// estimation sum /// estimation sum
...@@ -145,22 +141,16 @@ namespace AMDiS { ...@@ -145,22 +141,16 @@ namespace AMDiS {
/// estmation maximum /// estmation maximum
double estMax; double estMax;
/** \brief /// If true, elements are marked only if the new value is bigger than
* If true, elements are marked only if the new value is bigger than /// the current marking.
* the current marking.
*/
bool maximumMarking; bool maximumMarking;
/** \brief /// Lower limit for error estimation, from which an element is marked for
* Lower limit for error estimation, from which an element is marked for /// refinement
* refinement
*/
double markRLimit; double markRLimit;
/** \brief /// Upper limit for error estimation, from which an element is marked for
* Upper limit for error estimation, from which an element is marked for /// coarsening
* coarsening
*/
double markCLimit; double markCLimit;
/// power in estimator norm /// power in estimator norm
......
...@@ -87,7 +87,8 @@ namespace AMDiS { ...@@ -87,7 +87,8 @@ namespace AMDiS {
createBoundaryDofFlag(0), createBoundaryDofFlag(0),
boundaryDofInfo(1), boundaryDofInfo(1),
meshAdaptivity(true), meshAdaptivity(true),
hasPeriodicBoundary(false) hasPeriodicBoundary(false),
printTimings(false)
{ {
FUNCNAME("MeshDistributor::ParalleDomainBase()"); FUNCNAME("MeshDistributor::ParalleDomainBase()");
...@@ -99,7 +100,7 @@ namespace AMDiS { ...@@ -99,7 +100,7 @@ namespace AMDiS {
Parameters::get(name + "->debug output dir", debugOutputDir); Parameters::get(name + "->debug output dir", debugOutputDir);
Parameters::get(name + "->repartition ith change", repartitionIthChange); Parameters::get(name + "->repartition ith change", repartitionIthChange);
Parameters::get(name + "->mesh adaptivity", meshAdaptivity); Parameters::get(name + "->mesh adaptivity", meshAdaptivity);
string partStr = "parmetis"; string partStr = "parmetis";
Parameters::get(name + "->partitioner", partStr); Parameters::get(name + "->partitioner", partStr);
...@@ -124,6 +125,8 @@ namespace AMDiS { ...@@ -124,6 +125,8 @@ namespace AMDiS {
Parameters::get(name + "->box partitioning", tmp); Parameters::get(name + "->box partitioning", tmp);
partitioner->setBoxPartitioning(static_cast<bool>(tmp)); partitioner->setBoxPartitioning(static_cast<bool>(tmp));
Parameters::get(name + "->print timings", printTimings);
TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str()); TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
} }
...@@ -1584,6 +1587,10 @@ namespace AMDiS { ...@@ -1584,6 +1587,10 @@ namespace AMDiS {
{ {
FUNCNAME("MeshDistributor::createBoundaryDofs()"); FUNCNAME("MeshDistributor::createBoundaryDofs()");
if (printTimings)
MPI::COMM_WORLD.Barrier();
double first = MPI::Wtime();
// === Create DOF communicator. === // === Create DOF communicator. ===
dofComm.init(0, levelData, feSpaces); dofComm.init(0, levelData, feSpaces);
...@@ -1596,52 +1603,57 @@ namespace AMDiS { ...@@ -1596,52 +1603,57 @@ namespace AMDiS {
// === If requested, create more information on communication DOFs. === // === If requested, create more information on communication DOFs. ===
if (!createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) if (createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) {
return;
int nLevels = levelData.getLevelNumber();
boundaryDofInfo.resize(nLevels);
for (unsigned int i = 0; i < feSpaces.size(); i++) {
const FiniteElemSpace *feSpace = feSpaces[i];
for (int level = 0; level < nLevels; level++) { int nLevels = levelData.getLevelNumber();
boundaryDofInfo.resize(nLevels);
// === Clear data. ===
for (int geo = FACE; geo >= VERTEX; geo--) for (unsigned int i = 0; i < feSpaces.size(); i++) {
boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear(); const FiniteElemSpace *feSpace = feSpaces[i];
// === Create send DOFs. === for (int level = 0; level < nLevels; level++) {
for (int geo = FACE; geo >= VERTEX; geo--) {
for (InteriorBoundary::iterator it(intBoundary.getOwn(), level); // === Clear data. ===
!it.end(); ++it) { for (int geo = FACE; geo >= VERTEX; geo--)
if (it->rankObj.subObj == geo) { boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs); // === Create send DOFs. ===
for (int geo = FACE; geo >= VERTEX; geo--) {
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS)) for (InteriorBoundary::iterator it(intBoundary.getOwn(), level);
boundaryDofInfo[level][feSpace]. !it.end(); ++it) {
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end()); if (it->rankObj.subObj == geo) {
DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
boundaryDofInfo[level][feSpace].
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
}
} }
} }
}
// === Create recv DOFs. ===
// === Create recv DOFs. === for (int geo = FACE; geo >= VERTEX; geo--) {
for (int geo = FACE; geo >= VERTEX; geo--) { for (InteriorBoundary::iterator it(intBoundary.getOther(), level);
for (InteriorBoundary::iterator it(intBoundary.getOther(), level); !it.end(); ++it) {
!it.end(); ++it) { if (it->rankObj.subObj == geo) {
if (it->rankObj.subObj == geo) { DofContainer dofs;
DofContainer dofs; it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS)) boundaryDofInfo[level][feSpace].
boundaryDofInfo[level][feSpace]. geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end()); }
} }
} }
} }
} }
} }
if (printTimings) {
MPI::COMM_WORLD.Barrier();
MSG("MESHDIST 01 (createBoundaryDofs) needed %.5f seconds\n", MPI::Wtime() - first);
}
} }
...@@ -1679,22 +1691,8 @@ namespace AMDiS { ...@@ -1679,22 +1691,8 @@ namespace AMDiS {
// === Update all registered DOF mapping objects. === // === Update all registered DOF mapping objects. ===
TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) { updateParallelDofMappings();
vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
dofMaps[i]->clear();
for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
dofMaps[i]->update();
#if (DEBUG != 0)
// dofMaps[i]->printInfo();
#endif
}
// === Create periodic DOF maps, if there are periodic boundaries. === // === Create periodic DOF maps, if there are periodic boundaries. ===
...@@ -1769,6 +1767,37 @@ namespace AMDiS { ...@@ -1769,6 +1767,37 @@ namespace AMDiS {
} }
void MeshDistributor::updateParallelDofMappings()
{
FUNCNAME("MeshDistributor::updateParallelDofMappings()");
if (printTimings)
MPI::COMM_WORLD.Barrier();
double first = MPI::Wtime();
TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
dofMaps[i]->clear();
for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
dofMaps[i]->update();
#if (DEBUG != 0)
// dofMaps[i]->printInfo();
#endif
}
if (printTimings) {
MPI::COMM_WORLD.Barrier();
MSG("MESHDIST 02 (update DOF mappings) needed %.5f seconds\n", MPI::Wtime() - first);
}
}
void MeshDistributor::updateLocalGlobalNumbering(ParallelDofMapping &dofMap, void MeshDistributor::updateLocalGlobalNumbering(ParallelDofMapping &dofMap,
const FiniteElemSpace *feSpace) const FiniteElemSpace *feSpace)
{ {
...@@ -1776,6 +1805,7 @@ namespace AMDiS { ...@@ -1776,6 +1805,7 @@ namespace AMDiS {
DofComm &dcom = dofMap.getDofComm(); DofComm &dcom = dofMap.getDofComm();
// === Get all DOFs in ranks partition. === // === Get all DOFs in ranks partition. ===
std::set<const DegreeOfFreedom*> rankDofSet; std::set<const DegreeOfFreedom*> rankDofSet;
...@@ -1792,14 +1822,14 @@ namespace AMDiS { ...@@ -1792,14 +1822,14 @@ namespace AMDiS {
!it.end(); it.nextRank()) !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof()) for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof()); nonRankDofs.insert(it.getDof());
for (unsigned int i = 0; i < rankDofs.size(); i++) for (DofContainer::iterator it = rankDofs.begin();
if (nonRankDofs.count(rankDofs[i]) == 0) it != rankDofs.end(); ++it) {
dofMap[feSpace].insertRankDof(*(rankDofs[i])); if (nonRankDofs.count(*it))
dofMap[feSpace].insertNonRankDof(**it);
for (DofContainerSet::iterator it = nonRankDofs.begin(); else
it != nonRankDofs.end(); ++it) dofMap[feSpace].insertRankDof(**it);
dofMap[feSpace].insertNonRankDof(**it); }
} }
......
...@@ -322,6 +322,9 @@ namespace AMDiS { ...@@ -322,6 +322,9 @@ namespace AMDiS {
return levelData; return levelData;
} }
/// Updates all registered parallel DOF mappings, see \ref dofMaps.
void updateParallelDofMappings();
void updateLocalGlobalNumbering(); void updateLocalGlobalNumbering();
/// Updates the local and global DOF numbering after the mesh has been /// Updates the local and global DOF numbering after the mesh has been
...@@ -583,6 +586,9 @@ namespace AMDiS { ...@@ -583,6 +586,9 @@ namespace AMDiS {
/// solver objects and must be updated automatically after mesh change. /// solver objects and must be updated automatically after mesh change.
vector<ParallelDofMapping*> dofMaps; vector<ParallelDofMapping*> dofMaps;
/// If true, detailed timings for benchmarking are printed.
bool printTimings;
public: public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all /// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to /// face DOFs, edge DOFs and to the last vertex DOFs will be set to
......
...@@ -161,7 +161,8 @@ namespace AMDiS { ...@@ -161,7 +161,8 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by /// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
/// the rank. /// the rank.
void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1) inline void insertRankDof(DegreeOfFreedom dof0,
DegreeOfFreedom dof1 = -1)
{ {
FUNCNAME("ComponentDofMap::insertRankDof()"); FUNCNAME("ComponentDofMap::insertRankDof()");
...@@ -176,7 +177,8 @@ namespace AMDiS { ...@@ -176,7 +177,8 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain /// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
/// but is owned by a different rank, thus it is part of an interior boundary. /// but is owned by a different rank, thus it is part of an interior boundary.
void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1) inline void insertNonRankDof(DegreeOfFreedom dof0,
DegreeOfFreedom dof1 = -1)
{ {
FUNCNAME("ComponentDofMap::insertNonRankDof()"); FUNCNAME("ComponentDofMap::insertNonRankDof()");
...@@ -281,7 +283,7 @@ namespace AMDiS { ...@@ -281,7 +283,7 @@ namespace AMDiS {
DofMap dofMap; DofMap dofMap;
/// Set of all DOFs that are in mapping but are not owned by the rank. /// Set of all DOFs that are in mapping but are not owned by the rank.
boost::container::flat_set<DegreeOfFreedom> nonRankDofs; boost::container::flat_set<DegreeOfFreedom> nonRankDofs;
/// If true, a global index mapping will be computed for all DOFs. /// If true, a global index mapping will be computed for all DOFs.
bool globalMapping; bool globalMapping;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment