Commit ff250c55 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Some performance issues.

parent 323d3c5c
......@@ -200,6 +200,13 @@ namespace AMDiS {
const char *file,
int line)
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#if (DEBUG == 0)
if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
return;
#endif
#endif
static int old_line = -1;
if (!out)
......@@ -229,6 +236,12 @@ namespace AMDiS {
void Msg::print_warn(const char *format, ...)
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#if (DEBUG == 0)
if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
return;
#endif
#endif
va_list arg;
char buff[255];
......
......@@ -64,11 +64,9 @@ namespace AMDiS {
/// destructor
virtual ~Marker() {}
/** \brief
* Marks element with newMark. If \ref maximumMarking is set, the element
* is marked only if the new mark is bigger than the old one. The return
* value specifies whether the element has been marked, or not.
*/
/// Marks element with newMark. If \ref maximumMarking is set, the element
/// is marked only if the new mark is bigger than the old one. The return
/// value specifies whether the element has been marked, or not.
inline void setMark(Element *el, char newMark)
{
char oldMark = el->getMark();
......@@ -120,7 +118,7 @@ namespace AMDiS {
return elMarkCoarsen;
}
// Returns \ref name of the Marker
/// Returns \ref name of the Marker
inline std::string getName() const
{
return name;
......@@ -133,10 +131,8 @@ namespace AMDiS {
/// Name of the scalar marker.
std::string name;
/** \brief
* Equal to -1 for scalar problems. Component number if marker is
* part of a vector valued marker.
*/
/// Equal to -1 for scalar problems. Component number if marker is
/// part of a vector valued marker.
int row;
/// estimation sum
......@@ -145,22 +141,16 @@ namespace AMDiS {
/// estmation maximum
double estMax;
/** \brief
* If true, elements are marked only if the new value is bigger than
* the current marking.
*/
/// If true, elements are marked only if the new value is bigger than
/// the current marking.
bool maximumMarking;
/** \brief
* Lower limit for error estimation, from which an element is marked for
* refinement
*/
/// Lower limit for error estimation, from which an element is marked for
/// refinement
double markRLimit;
/** \brief
* Upper limit for error estimation, from which an element is marked for
* coarsening
*/
/// Upper limit for error estimation, from which an element is marked for
/// coarsening
double markCLimit;
/// power in estimator norm
......
......@@ -87,7 +87,8 @@ namespace AMDiS {
createBoundaryDofFlag(0),
boundaryDofInfo(1),
meshAdaptivity(true),
hasPeriodicBoundary(false)
hasPeriodicBoundary(false),
printTimings(false)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
......@@ -99,7 +100,7 @@ namespace AMDiS {
Parameters::get(name + "->debug output dir", debugOutputDir);
Parameters::get(name + "->repartition ith change", repartitionIthChange);
Parameters::get(name + "->mesh adaptivity", meshAdaptivity);
string partStr = "parmetis";
Parameters::get(name + "->partitioner", partStr);
......@@ -124,6 +125,8 @@ namespace AMDiS {
Parameters::get(name + "->box partitioning", tmp);
partitioner->setBoxPartitioning(static_cast<bool>(tmp));
Parameters::get(name + "->print timings", printTimings);
TEST_EXIT(partitioner)("Could not create partitioner \"%s\"!\n", partStr.c_str());
}
......@@ -1584,6 +1587,10 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createBoundaryDofs()");
if (printTimings)
MPI::COMM_WORLD.Barrier();
double first = MPI::Wtime();
// === Create DOF communicator. ===
dofComm.init(0, levelData, feSpaces);
......@@ -1596,52 +1603,57 @@ namespace AMDiS {
// === If requested, create more information on communication DOFs. ===
if (!createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED))
return;
int nLevels = levelData.getLevelNumber();
boundaryDofInfo.resize(nLevels);
for (unsigned int i = 0; i < feSpaces.size(); i++) {
const FiniteElemSpace *feSpace = feSpaces[i];
if (createBoundaryDofFlag.isSet(BOUNDARY_SUBOBJ_SORTED)) {
for (int level = 0; level < nLevels; level++) {
// === Clear data. ===
for (int geo = FACE; geo >= VERTEX; geo--)
boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
int nLevels = levelData.getLevelNumber();
boundaryDofInfo.resize(nLevels);
for (unsigned int i = 0; i < feSpaces.size(); i++) {
const FiniteElemSpace *feSpace = feSpaces[i];
// === Create send DOFs. ===
for (int geo = FACE; geo >= VERTEX; geo--) {
for (InteriorBoundary::iterator it(intBoundary.getOwn(), level);
!it.end(); ++it) {
if (it->rankObj.subObj == geo) {
DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
boundaryDofInfo[level][feSpace].
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
for (int level = 0; level < nLevels; level++) {
// === Clear data. ===
for (int geo = FACE; geo >= VERTEX; geo--)
boundaryDofInfo[level][feSpace].geoDofs[static_cast<GeoIndex>(geo)].clear();
// === Create send DOFs. ===
for (int geo = FACE; geo >= VERTEX; geo--) {
for (InteriorBoundary::iterator it(intBoundary.getOwn(), level);
!it.end(); ++it) {
if (it->rankObj.subObj == geo) {
DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_SEND_DOFS))
boundaryDofInfo[level][feSpace].
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
}
}
}
}
// === Create recv DOFs. ===
for (int geo = FACE; geo >= VERTEX; geo--) {
for (InteriorBoundary::iterator it(intBoundary.getOther(), level);
!it.end(); ++it) {
if (it->rankObj.subObj == geo) {
DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
boundaryDofInfo[level][feSpace].
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
// === Create recv DOFs. ===
for (int geo = FACE; geo >= VERTEX; geo--) {
for (InteriorBoundary::iterator it(intBoundary.getOther(), level);
!it.end(); ++it) {
if (it->rankObj.subObj == geo) {
DofContainer dofs;
it->rankObj.el->getAllDofs(feSpace, it->rankObj, dofs);
if (createBoundaryDofFlag.isSet(BOUNDARY_FILL_INFO_RECV_DOFS))
boundaryDofInfo[level][feSpace].
geoDofs[static_cast<GeoIndex>(geo)].insert(dofs.begin(), dofs.end());
}
}
}
}
}
}
if (printTimings) {
MPI::COMM_WORLD.Barrier();
MSG("MESHDIST 01 (createBoundaryDofs) needed %.5f seconds\n", MPI::Wtime() - first);
}
}
......@@ -1679,22 +1691,8 @@ namespace AMDiS {
// === Update all registered DOF mapping objects. ===
TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
dofMaps[i]->clear();
for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
dofMaps[i]->update();
#if (DEBUG != 0)
// dofMaps[i]->printInfo();
#endif
}
updateParallelDofMappings();
// === Create periodic DOF maps, if there are periodic boundaries. ===
......@@ -1769,6 +1767,37 @@ namespace AMDiS {
}
void MeshDistributor::updateParallelDofMappings()
{
FUNCNAME("MeshDistributor::updateParallelDofMappings()");
if (printTimings)
MPI::COMM_WORLD.Barrier();
double first = MPI::Wtime();
TEST_EXIT(dofMaps.size())("No DOF mapping defined!\n");
for (int i = 0; i < static_cast<int>(dofMaps.size()); i++) {
vector<const FiniteElemSpace*>& dofMapSpaces = dofMaps[i]->getFeSpaces();
dofMaps[i]->clear();
for (int j = 0; j < static_cast<int>(dofMapSpaces.size()); j++)
updateLocalGlobalNumbering(*(dofMaps[i]), dofMapSpaces[j]);
dofMaps[i]->update();
#if (DEBUG != 0)
// dofMaps[i]->printInfo();
#endif
}
if (printTimings) {
MPI::COMM_WORLD.Barrier();
MSG("MESHDIST 02 (update DOF mappings) needed %.5f seconds\n", MPI::Wtime() - first);
}
}
void MeshDistributor::updateLocalGlobalNumbering(ParallelDofMapping &dofMap,
const FiniteElemSpace *feSpace)
{
......@@ -1776,6 +1805,7 @@ namespace AMDiS {
DofComm &dcom = dofMap.getDofComm();
// === Get all DOFs in ranks partition. ===
std::set<const DegreeOfFreedom*> rankDofSet;
......@@ -1792,14 +1822,14 @@ namespace AMDiS {
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof());
for (unsigned int i = 0; i < rankDofs.size(); i++)
if (nonRankDofs.count(rankDofs[i]) == 0)
dofMap[feSpace].insertRankDof(*(rankDofs[i]));
for (DofContainerSet::iterator it = nonRankDofs.begin();
it != nonRankDofs.end(); ++it)
dofMap[feSpace].insertNonRankDof(**it);
for (DofContainer::iterator it = rankDofs.begin();
it != rankDofs.end(); ++it) {
if (nonRankDofs.count(*it))
dofMap[feSpace].insertNonRankDof(**it);
else
dofMap[feSpace].insertRankDof(**it);
}
}
......
......@@ -322,6 +322,9 @@ namespace AMDiS {
return levelData;
}
/// Updates all registered parallel DOF mappings, see \ref dofMaps.
void updateParallelDofMappings();
void updateLocalGlobalNumbering();
/// Updates the local and global DOF numbering after the mesh has been
......@@ -583,6 +586,9 @@ namespace AMDiS {
/// solver objects and must be updated automatically after mesh change.
vector<ParallelDofMapping*> dofMaps;
/// If true, detailed timings for benchmarking are printed.
bool printTimings;
public:
/// The boundary DOFs are sorted by subobject entities, i.e., first all
/// face DOFs, edge DOFs and to the last vertex DOFs will be set to
......
......@@ -161,7 +161,8 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
/// the rank.
void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
inline void insertRankDof(DegreeOfFreedom dof0,
DegreeOfFreedom dof1 = -1)
{
FUNCNAME("ComponentDofMap::insertRankDof()");
......@@ -176,7 +177,8 @@ namespace AMDiS {
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
/// but is owned by a different rank, thus it is part of an interior boundary.
void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
inline void insertNonRankDof(DegreeOfFreedom dof0,
DegreeOfFreedom dof1 = -1)
{
FUNCNAME("ComponentDofMap::insertNonRankDof()");
......@@ -281,7 +283,7 @@ namespace AMDiS {
DofMap dofMap;
/// Set of all DOFs that are in mapping but are not owned by the rank.
boost::container::flat_set<DegreeOfFreedom> nonRankDofs;
boost::container::flat_set<DegreeOfFreedom> nonRankDofs;
/// If true, a global index mapping will be computed for all DOFs.
bool globalMapping;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment