Skip to content

GitLab

Commit 5133dfb3 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Added more timing information to parallel computations.

parent ca3a8888
Hide whitespace changes
Inline Side-by-side
Showing with 160 additions and 79 deletions
AMDiS/src/AdaptInstationary.cc
View file @ 5133dfb3
......@@ -35,7 +35,6 @@ namespace AMDiS {
// MSG("Please use the constructor that uses references instead of pointers!\n");
initConstructor(problemStat, info, initialInfo, initialTimestampSet);
}
......@@ -52,7 +51,6 @@ namespace AMDiS {
FUNCNAME("AdaptInstationary::AdaptInstationary()");
initConstructor(&problemStat, &info, &initialInfo, initialTimestampSet);
}
......
AMDiS/src/Global.cc
View file @ 5133dfb3
......@@ -24,11 +24,16 @@
namespace AMDiS {
const char *funcName = NULL;
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
bool Msg::outputMainRank = false;
#endif
const char *Msg::oldFuncName = NULL;
std::ostream* Msg::out = NULL;
std::ostream* Msg::error = NULL;
int Global::dimOfWorld = 0;
std::vector< std::vector< int > > Global::geoIndexTable;
std::vector<std::vector<int> > Global::geoIndexTable;
int Msg::msgInfo = 10;
bool Msg::msgWait = true;
......@@ -39,6 +44,7 @@ namespace AMDiS {
new Tetrahedron(NULL)
};
void Msg::wait(bool w)
{
FUNCNAME("Msg::wait()");
......@@ -50,6 +56,7 @@ namespace AMDiS {
}
}
void Msg::change_out(std::ostream *fp)
{
FUNCNAME("Msg::change_out()");
......@@ -67,12 +74,13 @@ namespace AMDiS {
}
}
void Msg::change_error_out(std::ofstream *fp)
{
FUNCNAME("Msg::change_error_out()");
if (fp) {
if (error && *error != std::cout && *error != std::cerr) {
if (error && *error != std::cout && *error != std::cerr) {
dynamic_cast< std::ofstream*>(error)->close();
delete error;
}
......@@ -84,13 +92,14 @@ namespace AMDiS {
}
}
void Msg::open_error_file(const char *filename, OPENMODE type)
{
FUNCNAME("Msg::open_error_file()");
std::ofstream *fp;
if (filename && (fp = new std::ofstream(filename, type))) {
if (error && *error != std::cout && *error != std::cerr) {
if (filename && (fp = new std::ofstream(filename, type))) {
if (error && *error != std::cout && *error != std::cerr) {
dynamic_cast< std::ofstream*>(error)->close();
delete error;
}
......@@ -105,8 +114,14 @@ namespace AMDiS {
}
}
void Msg::print_funcname(const char *funcName)
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
return;
#endif
if (!out)
out = &std::cout;
......@@ -120,6 +135,7 @@ namespace AMDiS {
oldFuncName = funcName;
}
void Msg::print_error_funcname(const char *funcName, const char *file, int line)
{
static int old_line = -1;
......@@ -129,12 +145,13 @@ namespace AMDiS {
std::stringstream oss;
if (funcName && oldFuncName != funcName)
if (funcName && oldFuncName != funcName) {
oss << funcName << ": ";
else if (!funcName) {
} else if (!funcName) {
if (line-old_line > 5)
oss << "*unknown function*";
}
if (oldFuncName != funcName) {
oss << "ERROR in " << file << ", line " << line << std::endl;;
oldFuncName = funcName;
......@@ -145,6 +162,7 @@ namespace AMDiS {
old_line = line;
}
void Msg::print_error_exit(const char *format, ...)
{
va_list arg;
......@@ -161,6 +179,7 @@ namespace AMDiS {
exit(1);
}
void Msg::print_error(const char *format, ...)
{
va_list arg;
......@@ -176,6 +195,7 @@ namespace AMDiS {
va_end(arg);
}
void Msg::print_warn_funcname(const char *funcName,
const char *file,
int line)
......@@ -206,6 +226,7 @@ namespace AMDiS {
old_line = line;
}
void Msg::print_warn(const char *format, ...)
{
va_list arg;
......@@ -223,6 +244,11 @@ namespace AMDiS {
void Msg::print(const char *format, ...)
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (outputMainRank && MPI::COMM_WORLD.Get_rank() != 0)
return;
#endif
va_list arg;
char buff[255];
......@@ -235,6 +261,7 @@ namespace AMDiS {
va_end(arg);
}
void Global::init()
{
int d = -1;
......@@ -281,6 +308,7 @@ namespace AMDiS {
Msg::setMsgWait(!(d == 0));
}
void Global::clear()
{
delete referenceElement[1];
......@@ -288,6 +316,7 @@ namespace AMDiS {
delete referenceElement[3];
}
int fac(int i)
{
if (i <= 1)
......@@ -296,25 +325,11 @@ namespace AMDiS {
return i * fac(i - 1);
}
void waitSec(int seconds)
{
clock_t endwait = clock () + seconds * CLOCKS_PER_SEC;
while (clock() < endwait) {}
}
std::string memSizeStr(int size)
{
std::string result;
// if (size > 1024) {
// if (size > 1024 * 1024) {
// result << size / (1024 * 1024) << " MByte";
// } else {
// result << size / 1024 << " KByte";
// }
// } else {
// result << size << " Byte";
// }
return result;
}
}
AMDiS/src/Global.h
View file @ 5133dfb3
......@@ -50,7 +50,7 @@
#include <time.h>
#if HAVE_PARALLEL_DOMAIN_AMDIS
#include "mpi.h"
#include <mpi.h>
#endif
#include "boost/tuple/tuple.hpp"
......@@ -243,6 +243,14 @@ namespace AMDiS {
return error;
}
public:
#if HAVE_PARALLEL_DOMAIN_AMDIS
/// In parallel computations, when this variable is true, only the 0 rank will
/// print messages to the output stream. Error messages and warnings are always
/// printed from all ranks.
static bool outputMainRank;
#endif
protected:
/// Message stram
static std::ostream *out;
......@@ -306,11 +314,11 @@ namespace AMDiS {
/// prints a message, if min(Msg::msgInfo, info) >= noinfo
#define INFO(info,noinfo) \
if (Msg::getMsgInfo()&&(std::min(Msg::getMsgInfo(),(info))>=(noinfo))) MSG
if (Msg::getMsgInfo() && (std::min(Msg::getMsgInfo(), (info)) >= (noinfo))) MSG
/// prints a message, if min(Msg::msgInfo, info) >= noinfo
#define PRINT_INFO(info,noinfo) \
if (Msg::getMsgInfo()&&(std::min(Msg::getMsgInfo(),(info))>=(noinfo))) Msg::print
if (Msg::getMsgInfo() && (std::min(Msg::getMsgInfo(), (info)) >= (noinfo))) Msg::print
/** \brief
......@@ -442,8 +450,6 @@ namespace AMDiS {
const int RescheduleErrorCode = 23;
std::string memSizeStr(int size);
/**
* \ingroup Assembler
* \brief
......
AMDiS/src/ProblemInstat.cc
View file @ 5133dfb3
......@@ -9,6 +9,9 @@
//
// See also license.opensource.txt in the distribution.
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
#include <mpi.h>
#endif
#include "ProblemInstat.h"
#include "io/FileWriter.h"
......@@ -23,11 +26,13 @@ namespace AMDiS {
ProblemInstat::~ProblemInstat()
{}
void ProblemInstat::initialize(Flag initFlag,
ProblemInstat *adoptProblem/* = NULL*/,
Flag adoptFlag /* = INIT_NOTHING*/)
{}
void ProblemInstat::solveInitialProblem(AdaptInfo *adaptInfo)
{
AdaptStationary initialAdapt((name + "->initial->adapt").c_str(),
......@@ -37,6 +42,7 @@ namespace AMDiS {
initialAdapt.adapt();
}
void ProblemInstatScal::transferInitialSolution(AdaptInfo *adaptInfo)
{
TEST_EXIT(adaptInfo->getTime() == adaptInfo->getStartTime())
......@@ -44,6 +50,7 @@ namespace AMDiS {
problemStat->writeFiles(adaptInfo, true);
}
void ProblemInstatVec::transferInitialSolution(AdaptInfo *adaptInfo)
{
TEST_EXIT(adaptInfo->getTime() == adaptInfo->getStartTime())
......@@ -59,12 +66,14 @@ namespace AMDiS {
oldSolution(NULL)
{}
ProblemInstatScal::ProblemInstatScal(std::string sname, ProblemScal& prob)
: ProblemInstat(sname, NULL),
problemStat(&prob),
oldSolution(NULL)
{}
ProblemInstatScal::ProblemInstatScal(std::string sname,
ProblemScal& prob, ProblemStatBase& initialProb)
: ProblemInstat(sname, &initialProb),
......@@ -72,11 +81,13 @@ namespace AMDiS {
oldSolution(NULL)
{}
ProblemInstatScal::~ProblemInstatScal()
{
delete oldSolution;
}
void ProblemInstatScal::initialize(Flag initFlag,
ProblemInstat *adoptProblem,
Flag adoptFlag)
......@@ -131,8 +142,15 @@ namespace AMDiS {
void ProblemInstatVec::closeTimestep(AdaptInfo *adaptInfo)
{
FUNCNAME("ProblemInstatVec::closeTimestep()");
bool force = (adaptInfo->getTime() >= adaptInfo->getEndTime());
problemStat->writeFiles(adaptInfo, force);
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
MSG("Computational time for timestep: %.5f seconds\n",
(MPI::Wtime() - lastTimepoint));
#endif
}
......@@ -222,6 +240,10 @@ namespace AMDiS {
void ProblemInstatVec::initTimestep(AdaptInfo *adaptInfo)
{
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
lastTimepoint = MPI::Wtime();
#endif
oldSolution->copy(*(problemStat->getSolution()));
}
......
AMDiS/src/ProblemInstat.h
View file @ 5133dfb3
......@@ -260,6 +260,12 @@ namespace AMDiS {
/// Solution of the last timestep.
SystemVector *oldSolution;
/// In parallel computations, we want to print the overall computational time
/// that is used for one timestep.
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double lastTimepoint;
#endif
};
}
......
AMDiS/src/ProblemVec.cc
View file @ 5133dfb3
......@@ -537,10 +537,14 @@ namespace AMDiS {
{
FUNCNAME("ProblemVec::estimate()");
clock_t first = clock();
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime();
#else
#ifdef _OPENMP
double wtime = omp_get_wtime();
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
if (computeExactError) {
......@@ -565,12 +569,17 @@ namespace AMDiS {
}
}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("estimation of the error needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()), omp_get_wtime() - wtime);
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("estimation of the error needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
#endif
}
......@@ -1081,28 +1090,35 @@ namespace AMDiS {
{
FUNCNAME("ProblemVec::writeFiles()");
clock_t first = clock();
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double first = MPI::Wtime();
#else
#ifdef _OPENMP
double wtime = omp_get_wtime();
double first = omp_get_wtime();
#else
clock_t first = clock();
#endif
#endif
int i;
int size = static_cast<int>(fileWriters.size());
#ifdef _OPENMP
#pragma omp parallel for schedule(static, 1)
#endif
for (i = 0; i < size; i++) {
for (int i = 0; i < size; i++) {
fileWriters[i]->writeFiles(adaptInfo, force);
}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
INFO(info, 8)("writeFiles needed %.5f seconds\n",
MPI::Wtime() - first);
#else
#ifdef _OPENMP
INFO(info, 8)("writeFiles needed %.5f seconds system time / %.5f seconds wallclock time\n",
TIME_USED(first, clock()),
omp_get_wtime() - wtime);
INFO(info, 8)("writeFiles needed %.5f seconds\n",
omp_get_wtime() - first);
#else
INFO(info, 8)("writeFiles needed %.5f seconds\n",
TIME_USED(first, clock()));
#endif
#endif
}
......
AMDiS/src/StandardProblemIteration.cc
View file @ 5133dfb3
......@@ -20,6 +20,7 @@ namespace AMDiS {
int StandardProblemIteration::info = 10;
void StandardProblemIteration::beginIteration(AdaptInfo *adaptInfo)
{
FUNCNAME("StandardProblemIteration::beginIteration()");
......@@ -29,6 +30,7 @@ namespace AMDiS {
INFO(info, 4)("=============================\n");
}
Flag StandardProblemIteration::oneIteration(AdaptInfo *adaptInfo, Flag toDo)
{
FUNCNAME("StandardProblemIteration::oneIteration()");
......@@ -47,6 +49,7 @@ namespace AMDiS {
return flag;
}
void StandardProblemIteration::endIteration(AdaptInfo *adaptInfo)
{
FUNCNAME("StandardProblemIteration::endIteration()");
......@@ -56,6 +59,7 @@ namespace AMDiS {
INFO(info, 4)("=============================\n");
}
Flag StandardProblemIteration::buildAndAdapt(AdaptInfo *adaptInfo, Flag toDo)
{
FUNCNAME("StandardProblemIteration::buildAndAdapt()");
......@@ -90,18 +94,22 @@ namespace AMDiS {
return flag;
}
std::string StandardProblemIteration::getName()
{
return problem->getName();
}
void StandardProblemIteration::serialize(std::ostream &out)
{
problem->serialize(out);
}
void StandardProblemIteration::deserialize(std::istream &in)
{
problem->deserialize(in);
}
};
AMDiS/src/parallel/MeshDistributor.cc
View file @ 5133dfb3
......@@ -85,8 +85,11 @@ namespace AMDiS {
repartitioningAllowed = (tmp > 0);
GET_PARAMETER(0, name + "->debug output dir", &debugOutputDir);
GET_PARAMETER(0, name + "->repartition ith change", "%d", &repartitionIthChange);
tmp = 0;
GET_PARAMETER(0, name + "->log main rank", "%d", &tmp);
Msg::outputMainRank = (tmp > 0);
}
......@@ -146,7 +149,10 @@ namespace AMDiS {
// and now partition the mesh
partitioner->fillCoarsePartitionVec(&oldPartitionVec);
partitioner->partition(elemWeights, INITIAL);
bool partitioningSucceed = partitioner->partition(elemWeights, INITIAL);
TEST_EXIT(partitioningSucceed)("Initial partitioning does not work!\n");
partitioner->fillCoarsePartitionVec(&partitionVec);
......@@ -509,6 +515,8 @@ namespace AMDiS {
debug::writeMesh(feSpace, -1, debugOutputDir + "before_check_mesh");
#endif
double first = MPI::Wtime();
// === If mesh has not been changed on all ranks, return. ===
int recvAllValues = 0;
......@@ -520,9 +528,7 @@ namespace AMDiS {
// === At least one rank mesh has been changed, so the boundaries must be ===
// === adapted to the new mesh structure. ===
clock_t first = clock();
do {
// To check the interior boundaries, the ownership of the boundaries is not
// important. Therefore, we add all boundaries to one boundary container.
......@@ -566,9 +572,6 @@ namespace AMDiS {
debug::writeMesh(feSpace, -1, debugOutputDir + "mesh");
#endif
INFO(info, 8)("Parallel mesh adaption needed %.5f seconds\n",
TIME_USED(first, clock()));
// === Because the mesh has been changed, update the DOF numbering and mappings. ===
updateLocalGlobalNumbering();
......@@ -579,6 +582,11 @@ namespace AMDiS {
createPeriodicMap();
INFO(info, 8)("Parallel mesh adaption needed %.5f seconds\n",
MPI::Wtime() - first);
// === The mesh has changed, so check if it is required to repartition the mesh. ===
nTimestepsAfterLastRepartitioning++;
......@@ -1003,6 +1011,8 @@ namespace AMDiS {
if (repartitioning == 0)
return;
double timePoint = MPI::Wtime();
#if (DEBUG != 0)
ParallelDebug::testDoubleDofs(mesh);
......@@ -1016,8 +1026,6 @@ namespace AMDiS {
repartCounter++;
}
MSG("USED-SIZE A: %d\n", mesh->getDofAdmin(0).getUsedDofs());
#endif
......@@ -1031,8 +1039,15 @@ namespace AMDiS {
}
partitioner->useLocalGlobalDofMap(&mapLocalGlobalDofs);
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
if (!partitioningSucceed) {
MSG("ParMETIS created empty partition!\n");
return;
}
oldPartitionVec = partitionVec;
partitioner->partition(elemWeights, ADAPTIVE_REPART, 1000.0);
// === Create map that maps macro element indices to pointers to the ===
......@@ -1209,8 +1224,6 @@ namespace AMDiS {
VtkWriter::writeFile(&tmpa, oss.str());
repartCounter++;
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
ParallelDebug::testAllElements(*this);
ParallelDebug::testDoubleDofs(mesh);
#endif
......@@ -1230,13 +1243,12 @@ namespace AMDiS {
ParallelDebug::testAllElements(*this);
ParallelDebug::testInteriorBoundary(*this);
debug::writeMesh(feSpace, -1, debugOutputDir + "macro_mesh");
ParallelDebug::printBoundaryInfo(*this);
MSG("Debug mode tests finished!\n");
#endif
ParallelDebug::printBoundaryInfo(*this);
MSG("Mesh repartitioning needed %.5f seconds\n", MPI::Wtime() - timePoint);
}
......
AMDiS/src/parallel/ParMetisPartitioner.cc
View file @ 5133dfb3
......@@ -252,7 +252,7 @@ namespace AMDiS {
}
void ParMetisPartitioner::partition(std::map<int, double> &elemWeights,
bool ParMetisPartitioner::partition(std::map<int, double> &elemWeights,
PartitionMode mode,
float itr)
{
......@@ -331,22 +331,8 @@ namespace AMDiS {
// === Scale element weights. ===
double weightSum = 0.0;
for (int i = 0; i < nElements; i++) {
for (int i = 0; i < nElements; i++)
wgts[i] = static_cast<int>(floatWgts[i] * scale