Commit 13a09f5b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fix a billion of very small bugs, most related to performance issues.

parent e2b785f0
...@@ -92,7 +92,7 @@ namespace AMDiS { ...@@ -92,7 +92,7 @@ namespace AMDiS {
#ifdef HAVE_BDDC_ML #ifdef HAVE_BDDC_ML
addCreator("bddcml", new BddcMlSolver::Creator); addCreator("bddcml", new BddcMlSolver::Creator);
#endif #endif
addCreator("petsc-stokes", new PetscSolverNavierStokes::Creator); addCreator("petsc-navierstokes", new PetscSolverNavierStokes::Creator);
#endif #endif
} }
......
...@@ -653,12 +653,8 @@ namespace AMDiS { ...@@ -653,12 +653,8 @@ namespace AMDiS {
{ {
FUNCNAME("Element::getAllDofs()"); FUNCNAME("Element::getAllDofs()");
int sb = dofs.size();
getNodeDofs(feSpace, bound, dofs, baseDofPtr); getNodeDofs(feSpace, bound, dofs, baseDofPtr);
// MSG(" -> getNodeDofs %d\n", dofs.size() - sb);
if (dofGeoIndex != NULL) { if (dofGeoIndex != NULL) {
// In the case dofGeoIndex should be filled, set all node DOFs to be // In the case dofGeoIndex should be filled, set all node DOFs to be
// vertex DOFs. // vertex DOFs.
...@@ -667,15 +663,9 @@ namespace AMDiS { ...@@ -667,15 +663,9 @@ namespace AMDiS {
(*dofGeoIndex)[i] = VERTEX; (*dofGeoIndex)[i] = VERTEX;
} }
if (feSpace->getBasisFcts()->getDegree() > 1) { if (feSpace->getBasisFcts()->getDegree() > 1)
sb = dofs.size();
getHigherOrderDofs(feSpace, bound, dofs, baseDofPtr, dofGeoIndex); getHigherOrderDofs(feSpace, bound, dofs, baseDofPtr, dofGeoIndex);
// MSG(" -> getHODofs %d\n", dofs.size() - sb);
}
if (dofGeoIndex) { if (dofGeoIndex) {
TEST_EXIT_DBG(dofs.size() == dofGeoIndex->size()) TEST_EXIT_DBG(dofs.size() == dofGeoIndex->size())
("Arrays do not fit together: %d %d\n", dofs.size(), dofGeoIndex->size()); ("Arrays do not fit together: %d %d\n", dofs.size(), dofGeoIndex->size());
......
...@@ -108,7 +108,7 @@ namespace AMDiS { ...@@ -108,7 +108,7 @@ namespace AMDiS {
string arhPrefix = ""; string arhPrefix = "";
map<int, int> elInRank; map<int, int> elInRank;
map<int, int> elCodeSize; map<int, int> elCodeSize;
readMetaData(filename, elInRank, elCodeSize, arhPrefix); int nProc = readMetaData(filename, elInRank, elCodeSize, arhPrefix);
// === Check which arh files must be read by current rank. === // === Check which arh files must be read by current rank. ===
...@@ -133,8 +133,12 @@ namespace AMDiS { ...@@ -133,8 +133,12 @@ namespace AMDiS {
for (std::set<int>::iterator it = readArhFiles.begin(); for (std::set<int>::iterator it = readArhFiles.begin();
it != readArhFiles.end(); ++it) { it != readArhFiles.end(); ++it) {
string arhFilename = string arhFilename = directory.native() + "/" + arhPrefix;
directory.native() + "/" + arhPrefix + "-p" + boost::lexical_cast<string>(*it) + "-.arh"; if (nProc == 1)
arhFilename += ".arh";
else
arhFilename += "-p" + boost::lexical_cast<string>(*it) + "-.arh";
MSG("ARH file read from: %s\n", arhFilename.c_str()); MSG("ARH file read from: %s\n", arhFilename.c_str());
readFile(arhFilename, mesh, vecs); readFile(arhFilename, mesh, vecs);
} }
......
...@@ -81,9 +81,11 @@ namespace AMDiS { ...@@ -81,9 +81,11 @@ namespace AMDiS {
writeSerializationFile(false), writeSerializationFile(false),
repartitioningAllowed(false), repartitioningAllowed(false),
repartitionIthChange(20), repartitionIthChange(20),
repartitioningWaitAfterFail(20),
nMeshChangesAfterLastRepartitioning(0), nMeshChangesAfterLastRepartitioning(0),
repartitioningCounter(0), repartitioningCounter(0),
repartitioningFailed(0), repartitioningFailed(0),
debugOutputDir(""), debugOutputDir(""),
lastMeshChangeIndex(0), lastMeshChangeIndex(0),
createBoundaryDofFlag(0), createBoundaryDofFlag(0),
...@@ -101,6 +103,7 @@ namespace AMDiS { ...@@ -101,6 +103,7 @@ namespace AMDiS {
Parameters::get(name + "->repartitioning", repartitioningAllowed); Parameters::get(name + "->repartitioning", repartitioningAllowed);
Parameters::get(name + "->debug output dir", debugOutputDir); Parameters::get(name + "->debug output dir", debugOutputDir);
Parameters::get(name + "->repartition ith change", repartitionIthChange); Parameters::get(name + "->repartition ith change", repartitionIthChange);
Parameters::get(name + "->repartition wait after fail", repartitioningWaitAfterFail);
Parameters::get(name + "->mesh adaptivity", meshAdaptivity); Parameters::get(name + "->mesh adaptivity", meshAdaptivity);
string partStr = "parmetis"; string partStr = "parmetis";
...@@ -366,8 +369,9 @@ namespace AMDiS { ...@@ -366,8 +369,9 @@ namespace AMDiS {
// and now partition the mesh // and now partition the mesh
bool partitioningSucceed = partitioner->partition(elemWeights, INITIAL); bool partitioningSucceed = partitioner->partition(elemWeights, INITIAL);
TEST_EXIT(partitioningSucceed)("Initial partitioning does not work!\n"); TEST_EXIT(partitioningSucceed)("Initial partitioning does not work!\n");
partitioner->createPartitionMap(partitionMap);
} }
partitioner->createPartitionMap(partitionMap);
} }
...@@ -1372,7 +1376,7 @@ namespace AMDiS { ...@@ -1372,7 +1376,7 @@ namespace AMDiS {
partitioner->partition(elemWeights, ADAPTIVE_REPART); partitioner->partition(elemWeights, ADAPTIVE_REPART);
if (!partitioningSucceed) { if (!partitioningSucceed) {
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Barrier();
repartitioningFailed = 20; repartitioningFailed = repartitioningWaitAfterFail;;
MSG("Mesh partitioner created empty partition!\n"); MSG("Mesh partitioner created empty partition!\n");
MSG("Mesh repartitioning needed %.5f seconds\n", MPI::Wtime() - timePoint); MSG("Mesh repartitioning needed %.5f seconds\n", MPI::Wtime() - timePoint);
return; return;
...@@ -1395,7 +1399,7 @@ namespace AMDiS { ...@@ -1395,7 +1399,7 @@ namespace AMDiS {
if (!partitioningSucceed || !partitioner->meshChanged()) { if (!partitioningSucceed || !partitioner->meshChanged()) {
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Barrier();
repartitioningFailed = 20; repartitioningFailed = repartitioningWaitAfterFail;;
MSG("Mesh repartitioning needed %.5f seconds\n", MPI::Wtime() - timePoint); MSG("Mesh repartitioning needed %.5f seconds\n", MPI::Wtime() - timePoint);
return; return;
} }
...@@ -1562,7 +1566,9 @@ namespace AMDiS { ...@@ -1562,7 +1566,9 @@ namespace AMDiS {
// update the number of elements, vertices, etc. of the mesh. // update the number of elements, vertices, etc. of the mesh.
mesh->removeMacroElements(deleteMacroElements, feSpaces); mesh->removeMacroElements(deleteMacroElements, feSpaces);
// === Remove double DOFs. === // === Remove double DOFs. ===
MeshManipulation meshManipulation(mesh); MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(feSpaces, newMacroEl, elObjDb); meshManipulation.deleteDoubleDofs(feSpaces, newMacroEl, elObjDb);
......
...@@ -536,6 +536,9 @@ namespace AMDiS { ...@@ -536,6 +536,9 @@ namespace AMDiS {
/// repartitionings. /// repartitionings.
int repartitionIthChange; int repartitionIthChange;
///
int repartitioningWaitAfterFail;
/// Counts the number of mesh changes after the last mesh repartitioning /// Counts the number of mesh changes after the last mesh repartitioning
/// was done. /// was done.
int nMeshChangesAfterLastRepartitioning; int nMeshChangesAfterLastRepartitioning;
......
...@@ -83,7 +83,6 @@ namespace AMDiS { ...@@ -83,7 +83,6 @@ namespace AMDiS {
feSpace->getMesh()->getDofIndexCoords(coords); feSpace->getMesh()->getDofIndexCoords(coords);
#endif #endif
// === Traverse all new elements and connect them to the overall mesh === // === Traverse all new elements and connect them to the overall mesh ===
// === structure by deleting all double DOFs on macro element's vertices, === // === structure by deleting all double DOFs on macro element's vertices, ===
// === edges and faces. === // === edges and faces. ===
...@@ -105,8 +104,8 @@ namespace AMDiS { ...@@ -105,8 +104,8 @@ namespace AMDiS {
if (elIt->elIndex == (*it)->getIndex()) if (elIt->elIndex == (*it)->getIndex())
continue; continue;
if (macrosProcessed.count(elIt->elIndex) == 1) { if (macrosProcessed.count(elIt->elIndex)) {
TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex) == 1) TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex))
("Should not happen!\n"); ("Should not happen!\n");
Element *el0 = (*it)->getElement(); Element *el0 = (*it)->getElement();
...@@ -134,7 +133,7 @@ namespace AMDiS { ...@@ -134,7 +133,7 @@ namespace AMDiS {
if (elIt->elIndex == (*it)->getIndex()) if (elIt->elIndex == (*it)->getIndex())
continue; continue;
if (macrosProcessed.count(elIt->elIndex) == 1) { if (macrosProcessed.count(elIt->elIndex)) {
TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex)) TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex))
("Should not happen!\n"); ("Should not happen!\n");
...@@ -152,7 +151,7 @@ namespace AMDiS { ...@@ -152,7 +151,7 @@ namespace AMDiS {
el1->getAllDofs(feSpace, b1, dofs1, true, &dofGeoIndex1); el1->getAllDofs(feSpace, b1, dofs1, true, &dofGeoIndex1);
#if (DEBUG != 0) #if (DEBUG != 0)
if (feSpaces.size() == 1) if (feSpaces.size())
debug::testDofsByCoords(coords, dofs0, dofs1); debug::testDofsByCoords(coords, dofs0, dofs1);
else else
TEST_EXIT_DBG(dofs0.size() == dofs1.size()) TEST_EXIT_DBG(dofs0.size() == dofs1.size())
...@@ -184,7 +183,7 @@ namespace AMDiS { ...@@ -184,7 +183,7 @@ namespace AMDiS {
if (elIt->elIndex == (*it)->getIndex()) if (elIt->elIndex == (*it)->getIndex())
continue; continue;
if (macrosProcessed.count(elIt->elIndex) == 1) { if (macrosProcessed.count(elIt->elIndex)) {
TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex)) TEST_EXIT_DBG(macroIndexMap.count(elIt->elIndex))
("Should not happen!\n"); ("Should not happen!\n");
...@@ -202,7 +201,7 @@ namespace AMDiS { ...@@ -202,7 +201,7 @@ namespace AMDiS {
el1->getAllDofs(feSpace, b1, dofs1, true, &dofGeoIndex1); el1->getAllDofs(feSpace, b1, dofs1, true, &dofGeoIndex1);
#if (DEBUG != 0) #if (DEBUG != 0)
if (feSpaces.size() == 1) if (feSpaces.size())
debug::testDofsByCoords(coords, dofs0, dofs1); debug::testDofsByCoords(coords, dofs0, dofs1);
else else
TEST_EXIT_DBG(dofs0.size() == dofs1.size()) TEST_EXIT_DBG(dofs0.size() == dofs1.size())
...@@ -355,7 +354,8 @@ namespace AMDiS { ...@@ -355,7 +354,8 @@ namespace AMDiS {
// === To the mesh structure code. === // === To the mesh structure code. ===
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, traverseFlag); ElInfo *elInfo =
stack.traverseFirstOneMacro(mesh, boundEl.elIndex, -1, traverseFlag);
if (s0 != -1) { if (s0 != -1) {
while (elInfo && elInfo->getElement() != child0) while (elInfo && elInfo->getElement() != child0)
......
...@@ -833,8 +833,7 @@ namespace AMDiS { ...@@ -833,8 +833,7 @@ namespace AMDiS {
map<int, double> vec; map<int, double> vec;
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, Mesh::CALL_LEAF_EL);
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) { while (elInfo) {
int index = elInfo->getElement()->getIndex(); int index = elInfo->getElement()->getIndex();
......
...@@ -250,7 +250,7 @@ namespace AMDiS { ...@@ -250,7 +250,7 @@ namespace AMDiS {
void setSolver(KSP ksp, void setSolver(KSP ksp,
string kspPrefix, const char* kspPrefix,
KSPType kspType, KSPType kspType,
PCType pcType, PCType pcType,
PetscReal rtol, PetscReal rtol,
...@@ -259,8 +259,9 @@ namespace AMDiS { ...@@ -259,8 +259,9 @@ namespace AMDiS {
{ {
KSPSetType(ksp, kspType); KSPSetType(ksp, kspType);
KSPSetTolerances(ksp, rtol, atol, PETSC_DEFAULT, maxIt); KSPSetTolerances(ksp, rtol, atol, PETSC_DEFAULT, maxIt);
if (kspPrefix != "") //if (kspPrefix != "")
KSPSetOptionsPrefix(ksp, kspPrefix.c_str()); if (strcmp(kspPrefix, "") != 0)
KSPSetOptionsPrefix(ksp, kspPrefix);
KSPSetFromOptions(ksp); KSPSetFromOptions(ksp);
PC pc; PC pc;
......
...@@ -92,7 +92,7 @@ namespace AMDiS { ...@@ -92,7 +92,7 @@ namespace AMDiS {
void matNestConvert(Mat matNest, Mat &mat); void matNestConvert(Mat matNest, Mat &mat);
void setSolver(KSP ksp, void setSolver(KSP ksp,
string kspPrefix, const char* kspPrefix,
KSPType kspType, KSPType kspType,
PCType pcType, PCType pcType,
PetscReal rtol = PETSC_DEFAULT, PetscReal rtol = PETSC_DEFAULT,
......
...@@ -501,20 +501,20 @@ namespace AMDiS { ...@@ -501,20 +501,20 @@ namespace AMDiS {
isNames[i].c_str()); isNames[i].c_str());
} }
createFieldSplit(pc, isNames[i], blockComponents); createFieldSplit(pc, isNames[i].c_str(), blockComponents);
} }
} }
void PetscSolverGlobalMatrix::createFieldSplit(PC pc, void PetscSolverGlobalMatrix::createFieldSplit(PC pc,
string splitName, const char* splitName,
vector<int> &components) vector<int> &components)
{ {
FUNCNAME("PetscSolverGlobalMatrix::createFieldSplit()"); FUNCNAME("PetscSolverGlobalMatrix::createFieldSplit()");
IS is; IS is;
interiorMap->createIndexSet(is, components[0], components.size()); interiorMap->createIndexSet(is, components[0], components.size());
PCFieldSplitSetIS(pc, splitName.c_str(), is); PCFieldSplitSetIS(pc, splitName, is);
ISDestroy(&is); ISDestroy(&is);
} }
...@@ -823,7 +823,7 @@ namespace AMDiS { ...@@ -823,7 +823,7 @@ namespace AMDiS {
fe.push_back(componentSpaces[component]); fe.push_back(componentSpaces[component]);
PetscSolver* subSolver = new PetscSolverGlobalMatrix(""); PetscSolver* subSolver = new PetscSolverGlobalMatrix("");
subSolver->setKspPrefix(kspPrefix.c_str()); subSolver->setKspPrefix(kspPrefix);
subSolver->setMeshDistributor(meshDistributor, subSolver->setMeshDistributor(meshDistributor,
mpiCommGlobal, mpiCommGlobal,
mpiCommLocal); mpiCommLocal);
......
...@@ -88,10 +88,10 @@ namespace AMDiS { ...@@ -88,10 +88,10 @@ namespace AMDiS {
* \param[in] components System component numbers of the field split. At * \param[in] components System component numbers of the field split. At
* the moment only continuous splits are allowed. * the moment only continuous splits are allowed.
*/ */
void createFieldSplit(PC pc, string splitName, vector<int> &components); void createFieldSplit(PC pc, const char* splitName, vector<int> &components);
/// Wrapper to create field split from only one component. /// Wrapper to create field split from only one component.
void createFieldSplit(PC pc, string splitName, int component) void createFieldSplit(PC pc, const char* splitName, int component)
{ {
vector<int> components; vector<int> components;
components.push_back(component); components.push_back(component);
......
...@@ -46,6 +46,7 @@ namespace AMDiS { ...@@ -46,6 +46,7 @@ namespace AMDiS {
: PetscSolverGlobalMatrix(name), : PetscSolverGlobalMatrix(name),
pressureComponent(-1), pressureComponent(-1),
pressureNullSpace(true), pressureNullSpace(true),
useOldInitialGuess(false),
massMatrixSolver(NULL), massMatrixSolver(NULL),
laplaceMatrixSolver(NULL), laplaceMatrixSolver(NULL),
nu(NULL), nu(NULL),
...@@ -61,6 +62,28 @@ namespace AMDiS { ...@@ -61,6 +62,28 @@ namespace AMDiS {
Parameters::get(initFileStr + "->navierstokes->pressure null space", Parameters::get(initFileStr + "->navierstokes->pressure null space",
pressureNullSpace); pressureNullSpace);
TEST_EXIT(pressureNullSpace)("This is not yet tested, may be wrong!\n"); TEST_EXIT(pressureNullSpace)("This is not yet tested, may be wrong!\n");
Parameters::get(initFileStr + "->navierstokes->use old initial guess",
useOldInitialGuess);
}
void PetscSolverNavierStokes::solvePetscMatrix(SystemVector &vec,
AdaptInfo *adaptInfo)
{
FUNCNAME("PetscSolverNavierStokes::solvePetscMatrix()");
if (useOldInitialGuess) {
VecSet(getVecSolInterior(), 0.0);
for (int i = 0; i < solution->getSize(); i++)
setDofVector(getVecSolInterior(), solution->getDOFVector(i), i, true);
vecSolAssembly();
KSPSetInitialGuessNonzero(kspInterior, PETSC_TRUE);
}
PetscSolverGlobalMatrix::solvePetscMatrix(vec, adaptInfo);
} }
...@@ -151,7 +174,7 @@ namespace AMDiS { ...@@ -151,7 +174,7 @@ namespace AMDiS {
else else
laplaceOp.addTerm(new VecAtQP_SOT(phase, &idFct)); laplaceOp.addTerm(new VecAtQP_SOT(phase, &idFct));
laplaceMatrix.assembleOperator(laplaceOp); laplaceMatrix.assembleOperator(laplaceOp);
laplaceMatrixSolver = createSubSolver(pressureComponent, "laplace_"); laplaceMatrixSolver = createSubSolver(pressureComponent, string("laplace_"));
laplaceMatrixSolver->fillPetscMatrix(&laplaceMatrix); laplaceMatrixSolver->fillPetscMatrix(&laplaceMatrix);
......
...@@ -120,8 +120,11 @@ namespace AMDiS { ...@@ -120,8 +120,11 @@ namespace AMDiS {
return new PetscSolverNavierStokes(this->name); return new PetscSolverNavierStokes(this->name);
} }
}; };
PetscSolverNavierStokes(string name); PetscSolverNavierStokes(string name);
void solvePetscMatrix(SystemVector &vec, AdaptInfo *adaptInfo);
void setStokesData(double *nuPtr, double *invTauPtr, SystemVector *vec) void setStokesData(double *nuPtr, double *invTauPtr, SystemVector *vec)
{ {
nu = nuPtr; nu = nuPtr;
...@@ -144,6 +147,9 @@ namespace AMDiS { ...@@ -144,6 +147,9 @@ namespace AMDiS {
bool pressureNullSpace; bool pressureNullSpace;
/// If true, old solution is used for initial guess in solver phase.
bool useOldInitialGuess;
PetscSolver *massMatrixSolver, *laplaceMatrixSolver, *conDifMatrixSolver; PetscSolver *massMatrixSolver, *laplaceMatrixSolver, *conDifMatrixSolver;
NavierStokesSchurData matShellContext; NavierStokesSchurData matShellContext;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment