Commit da4acc42 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Add support for BDDCML 2.0 and work on scalable FETI-DP in 3d using arithmetic constraints.

parent f99a87c8
...@@ -26,9 +26,6 @@ ...@@ -26,9 +26,6 @@
#include "BoundaryManager.h" #include "BoundaryManager.h"
#include "Assembler.h" #include "Assembler.h"
#include "Serializer.h" #include "Serializer.h"
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#include "parallel/ParallelDofMapping.h"
#endif
namespace AMDiS { namespace AMDiS {
...@@ -209,21 +206,6 @@ namespace AMDiS { ...@@ -209,21 +206,6 @@ namespace AMDiS {
using namespace mtl; using namespace mtl;
#if 0
if (MPI::COMM_WORLD.Get_rank() == 0) {
std::cout << "----- PRINT MAT " << rowElInfo->getElement()->getIndex() << "--------" << std::endl;
std::cout << elMat << std::endl;
std::cout << "rows: ";
for (int i = 0; i < rowIndices.size(); i++)
std::cout << rowIndices[i] << " ";
std::cout << std::endl;
std::cout << "cols: ";
for (int i = 0; i < colIndices.size(); i++)
std::cout << colIndices[i] << " ";
std::cout << std::endl;
}
#endif
for (int i = 0; i < nRow; i++) { for (int i = 0; i < nRow; i++) {
DegreeOfFreedom row = rowIndices[i]; DegreeOfFreedom row = rowIndices[i];
...@@ -552,15 +534,12 @@ namespace AMDiS { ...@@ -552,15 +534,12 @@ namespace AMDiS {
// Do the following only in sequential code. In parallel mode, the specific // Do the following only in sequential code. In parallel mode, the specific
// solver method must care about dirichlet boundary conditions. // solver method must care about dirichlet boundary conditions.
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
inserter_type &ins = *inserter; inserter_type &ins = *inserter;
for (std::set<int>::iterator it = dirichletDofs.begin(); for (std::set<int>::iterator it = dirichletDofs.begin();
it != dirichletDofs.end(); ++it) it != dirichletDofs.end(); ++it)
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS ins[*it][*it] = 1.0;
// if (dofMap->isRankDof(*it))
if (dofMap->isSet(*it))
#endif #endif
ins[*it][*it] = 1.0;
} }
...@@ -583,12 +562,4 @@ namespace AMDiS { ...@@ -583,12 +562,4 @@ namespace AMDiS {
dirichletDofs.clear(); dirichletDofs.clear();
} }
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
void DOFMatrix::setDofMap(FeSpaceDofMap& m)
{
dofMap = &m;
}
#endif
} }
...@@ -415,10 +415,6 @@ namespace AMDiS { ...@@ -415,10 +415,6 @@ namespace AMDiS {
/// ///
int memsize(); int memsize();
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
void setDofMap(FeSpaceDofMap& m);
#endif
protected: protected:
/// Pointer to a FiniteElemSpace with information about corresponding row DOFs /// Pointer to a FiniteElemSpace with information about corresponding row DOFs
/// and basis functions /// and basis functions
...@@ -480,13 +476,6 @@ namespace AMDiS { ...@@ -480,13 +476,6 @@ namespace AMDiS {
/// elemnts during assembling. /// elemnts during assembling.
int nnzPerRow; int nnzPerRow;
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
/// Is used in parallel computations to check whether specific DOFs in the
/// row FE spaces are owned by the rank or not. This is used to ensure that
/// Dirichlet BC is handled correctly in parallel computations.
FeSpaceDofMap *dofMap;
#endif
/// Inserter object: implemented as pointer, allocated and deallocated as needed /// Inserter object: implemented as pointer, allocated and deallocated as needed
inserter_type *inserter; inserter_type *inserter;
......
...@@ -285,10 +285,10 @@ namespace AMDiS { ...@@ -285,10 +285,10 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS #ifdef HAVE_PARALLEL_DOMAIN_AMDIS
double localResult = result; double localResult = result;
MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM); MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM);
#endif #endif
return result; return result;
} }
......
...@@ -42,10 +42,6 @@ ...@@ -42,10 +42,6 @@
#include "BasisFunction.h" #include "BasisFunction.h"
#include "FiniteElemSpace.h" #include "FiniteElemSpace.h"
#include "SurfaceQuadrature.h" #include "SurfaceQuadrature.h"
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#include <mpi.h>
#include "parallel/ParallelDofMapping.h"
#endif
namespace AMDiS { namespace AMDiS {
...@@ -61,11 +57,7 @@ namespace AMDiS { ...@@ -61,11 +57,7 @@ namespace AMDiS {
elementVector(3), elementVector(3),
boundaryManager(NULL), boundaryManager(NULL),
nBasFcts(0) nBasFcts(0)
{ {}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
dofMap = NULL;
#endif
}
DOFVectorBase(const FiniteElemSpace *f, string n); DOFVectorBase(const FiniteElemSpace *f, string n);
...@@ -248,21 +240,6 @@ namespace AMDiS { ...@@ -248,21 +240,6 @@ namespace AMDiS {
return dirichletDofValues; return dirichletDofValues;
} }
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
void setDofMap(FeSpaceDofMap& m)
{
dofMap = &m;
}
inline bool isRankDof(DegreeOfFreedom dof)
{
TEST_EXIT_DBG(dofMap)("No rank dofs set!\n");
return dofMap->isRankDof(dof);
}
#endif
protected: protected:
/// ///
const FiniteElemSpace *feSpace; const FiniteElemSpace *feSpace;
...@@ -292,10 +269,6 @@ namespace AMDiS { ...@@ -292,10 +269,6 @@ namespace AMDiS {
int dim; int dim;
map<DegreeOfFreedom, T> dirichletDofValues; map<DegreeOfFreedom, T> dirichletDofValues;
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
FeSpaceDofMap *dofMap;
#endif
}; };
......
...@@ -1060,10 +1060,6 @@ namespace AMDiS { ...@@ -1060,10 +1060,6 @@ namespace AMDiS {
this->boundaryManager = NULL; this->boundaryManager = NULL;
} }
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
this->dofMap = rhs.dofMap;
#endif
return *this; return *this;
} }
......
...@@ -64,24 +64,20 @@ namespace AMDiS { ...@@ -64,24 +64,20 @@ namespace AMDiS {
const BasisFunction *basFcts = rowFeSpace->getBasisFcts(); const BasisFunction *basFcts = rowFeSpace->getBasisFcts();
for (int i = 0; i < nBasFcts; i++) { for (int i = 0; i < nBasFcts; i++) {
if (localBound[i] == boundaryType) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS double value = 0.0;
if (vector->isRankDof(dofIndices[i])) if (f) {
#endif elInfo->coordToWorld(*(basFcts->getCoords(i)), worldCoords);
if (localBound[i] == boundaryType) { value = (*f)(worldCoords);
double value = 0.0; } else {
if (f) { if (dofVec)
elInfo->coordToWorld(*(basFcts->getCoords(i)), worldCoords); value = (*dofVec)[dofIndices[i]];
value = (*f)(worldCoords); else
} else { ERROR_EXIT("There is something wrong!\n");
if (dofVec)
value = (*dofVec)[dofIndices[i]];
else
ERROR_EXIT("There is something wrong!\n");
}
vector->setDirichletDofValue(dofIndices[i], value);
(*vector)[dofIndices[i]] = value;
} }
vector->setDirichletDofValue(dofIndices[i], value);
(*vector)[dofIndices[i]] = value;
}
} }
} }
......
...@@ -232,6 +232,7 @@ namespace AMDiS { ...@@ -232,6 +232,7 @@ namespace AMDiS {
// matrix type (set here to unsymmetric) // matrix type (set here to unsymmetric)
int matrixtype = 0; int matrixtype = 0;
Parameters::get("parallel->bddcml->matrix type", matrixtype);
// Non zero structure of matrix // Non zero structure of matrix
vector<int> i_sparse; vector<int> i_sparse;
...@@ -352,12 +353,14 @@ namespace AMDiS { ...@@ -352,12 +353,14 @@ namespace AMDiS {
&luser_constraints2); &luser_constraints2);
int use_defaults_int = 1; int use_defaults_int = 0;
int parallel_division_int = 1; int parallel_division_int = 1;
int use_arithmetic_int = 0; int use_arithmetic_int = 1;
int use_adaptive_int = 0; int use_adaptive_int = 0;
int use_user_constraint_int = 0; int use_user_constraint_int = 0;
Parameters::get("parallel->bddcml->arithmetic constraints", use_arithmetic_int);
// MSG("call to \"bddcml_setup_preconditioner\" with the following arguments (each in one line):\n"); // MSG("call to \"bddcml_setup_preconditioner\" with the following arguments (each in one line):\n");
// MSG(" %d\n", matrixtype); // MSG(" %d\n", matrixtype);
// MSG(" %d\n", use_defaults_int); // MSG(" %d\n", use_defaults_int);
......
...@@ -189,8 +189,6 @@ namespace AMDiS { ...@@ -189,8 +189,6 @@ namespace AMDiS {
updateLocalGlobalNumbering(); updateLocalGlobalNumbering();
setRankDofs();
elObjDb.setData(partitionMap, levelData); elObjDb.setData(partitionMap, levelData);
#if (DEBUG != 0) #if (DEBUG != 0)
...@@ -351,9 +349,6 @@ namespace AMDiS { ...@@ -351,9 +349,6 @@ namespace AMDiS {
#endif #endif
} }
// Set DOF rank information to all matrices and vectors.
setRankDofs();
// And delete some data, we there is no mesh adaptivty. // And delete some data, we there is no mesh adaptivty.
if (!meshAdaptivity) if (!meshAdaptivity)
elObjDb.clear(); elObjDb.clear();
...@@ -475,10 +470,8 @@ namespace AMDiS { ...@@ -475,10 +470,8 @@ namespace AMDiS {
// DOFs object to the matrices and vectors of the added stationary problem, // DOFs object to the matrices and vectors of the added stationary problem,
// and to remove the periodic boundary conditions on these objects. // and to remove the periodic boundary conditions on these objects.
if (initialized) { if (initialized)
setRankDofs(probStat, dofMap);
removePeriodicBoundaryConditions(probStat); removePeriodicBoundaryConditions(probStat);
}
} }
...@@ -731,49 +724,6 @@ namespace AMDiS { ...@@ -731,49 +724,6 @@ namespace AMDiS {
} }
void MeshDistributor::setRankDofs(ProblemStatSeq *probStat,
ParallelDofMapping &rankDofMap)
{
FUNCNAME("MeshDistributor::setRankDofs()");
int nComponents = probStat->getNumComponents();
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++)
if (probStat->getSystemMatrix(i, j)) {
const FiniteElemSpace *rowFeSpace =
probStat->getSystemMatrix(i, j)->getRowFeSpace();
TEST_EXIT(find(feSpaces.begin(), feSpaces.end(), rowFeSpace) != feSpaces.end())
("Should not happen!\n");
probStat->getSystemMatrix(i, j)->setDofMap(rankDofMap[rowFeSpace]);
}
TEST_EXIT_DBG(probStat->getRhs()->getDOFVector(i))("No RHS vector!\n");
TEST_EXIT_DBG(probStat->getSolution()->getDOFVector(i))
("No solution vector!\n");
TEST_EXIT_DBG(probStat->getRhsVector(i)->getFeSpace() ==
probStat->getSolution(i)->getFeSpace())
("Should not happen!\n");
const FiniteElemSpace *feSpace = probStat->getRhsVector(i)->getFeSpace();
TEST_EXIT(find(feSpaces.begin(), feSpaces.end(), feSpace) != feSpaces.end())
("Should not happen!\n");
probStat->getRhsVector(i)->setDofMap(rankDofMap[feSpace]);
probStat->getSolution(i)->setDofMap(rankDofMap[feSpace]);
}
}
void MeshDistributor::setRankDofs()
{
for (unsigned int i = 0; i < problemStat.size(); i++)
setRankDofs(problemStat[i], dofMap);
}
void MeshDistributor::removePeriodicBoundaryConditions() void MeshDistributor::removePeriodicBoundaryConditions()
{ {
FUNCNAME("MeshDistributor::removePeriodicBoundaryConditions()"); FUNCNAME("MeshDistributor::removePeriodicBoundaryConditions()");
......
...@@ -347,14 +347,6 @@ namespace AMDiS { ...@@ -347,14 +347,6 @@ namespace AMDiS {
DofComm &dcom, DofComm &dcom,
const FiniteElemSpace *feSpace); const FiniteElemSpace *feSpace);
/// Sets \ref isRankDof to all matrices and rhs vectors in a given
/// stationary problem.
void setRankDofs(ProblemStatSeq *probStat, ParallelDofMapping &rankDofMap);
/// Sets \ref isRankDof to all matrices and rhs vectors in all
/// stationary problems.
void setRankDofs();
protected: protected:
void addProblemStat(ProblemStatSeq *probStat); void addProblemStat(ProblemStatSeq *probStat);
......
...@@ -76,24 +76,6 @@ namespace AMDiS { ...@@ -76,24 +76,6 @@ namespace AMDiS {
} }
void PetscProblemStat::buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
bool assembleVector)
{
FUNCNAME("ParallelProblemStatBase::buildAfterCoarsen()");
TEST_EXIT_DBG(MeshDistributor::globalMeshDistributor != NULL)
("Should not happen!\n");
MeshDistributor::globalMeshDistributor->checkMeshChange();
MeshDistributor::globalMeshDistributor->setRankDofs(this,
petscSolver->getInteriorMap());
ProblemStatSeq::buildAfterCoarsen(adaptInfo, flag,
assembleMatrix,
assembleVector);
}
void PetscProblemStat::solve(AdaptInfo *adaptInfo, void PetscProblemStat::solve(AdaptInfo *adaptInfo,
bool createMatrixData, bool createMatrixData,
bool storeMatrixData) bool storeMatrixData)
......
...@@ -49,10 +49,6 @@ namespace AMDiS { ...@@ -49,10 +49,6 @@ namespace AMDiS {
delete petscSolver; delete petscSolver;
} }
void buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
bool assembleVector);
void initialize(Flag initFlag, void initialize(Flag initFlag,
ProblemStatSeq *adoptProblem = NULL, ProblemStatSeq *adoptProblem = NULL,
Flag adoptFlag = INIT_NOTHING); Flag adoptFlag = INIT_NOTHING);
......
...@@ -23,9 +23,10 @@ namespace AMDiS { ...@@ -23,9 +23,10 @@ namespace AMDiS {
PetscSolver::PetscSolver() PetscSolver::PetscSolver()
: ParallelCoarseSpaceMatVec(), : ParallelCoarseSpaceMatVec(),
kspPrefix(""), kspPrefix(""),
removeRhsNullspace(false), removeRhsNullspace(false),
hasConstantNullspace(false),
isSymmetric(false), isSymmetric(false),
hasConstantNullspace(false) handleDirichletRows(true)
{ {
string kspStr = ""; string kspStr = "";
Parameters::get("parallel->solver->petsc->ksp", kspStr); Parameters::get("parallel->solver->petsc->ksp", kspStr);
......
...@@ -62,7 +62,8 @@ namespace AMDiS { ...@@ -62,7 +62,8 @@ namespace AMDiS {
*/ */
virtual void fillPetscMatrix(Matrix<DOFMatrix*> *mat) = 0; virtual void fillPetscMatrix(Matrix<DOFMatrix*> *mat) = 0;
/// /// This function is just a small wrapper that creates a 1x1 matrix that
/// contains exactly one DOFMatrix and than calls \ref fillPetscMatrix
void fillPetscMatrix(DOFMatrix* mat); void fillPetscMatrix(DOFMatrix* mat);
/** \brief /** \brief
...@@ -85,11 +86,6 @@ namespace AMDiS { ...@@ -85,11 +86,6 @@ namespace AMDiS {
/// Detroys all vector data structures. /// Detroys all vector data structures.
virtual void destroyVectorData() = 0; virtual void destroyVectorData() = 0;
virtual ParallelDofMapping &getInteriorMap()
{
return meshDistributor->getDofMap();
}
virtual Flag getBoundaryDofRequirement() virtual Flag getBoundaryDofRequirement()
{ {
return 0; return 0;
...@@ -139,6 +135,12 @@ namespace AMDiS { ...@@ -139,6 +135,12 @@ namespace AMDiS {
constNullspaceComponent.push_back(component); constNullspaceComponent.push_back(component);
} }
/// Informs the solver whether is has to handle dirichlet rows or not.
void setHandleDirichletRows(bool b)
{
handleDirichletRows = b;
}
protected: protected:
/** \brief /** \brief
* Copies between to PETSc vectors by using different index sets for the * Copies between to PETSc vectors by using different index sets for the
...@@ -177,6 +179,12 @@ namespace AMDiS { ...@@ -177,6 +179,12 @@ namespace AMDiS {
bool isSymmetric; bool isSymmetric;
/// If true, dirichlet rows are handled by the solver correspondently. To
/// set this value to false makes only sense, of this solver is just used
/// as a subsolver and the main solver above alread handles dirichlet rows
/// in some way.
bool handleDirichletRows;
vector<int> constNullspaceComponent; vector<int> constNullspaceComponent;
}; };
......
...@@ -101,6 +101,7 @@ namespace AMDiS { ...@@ -101,6 +101,7 @@ namespace AMDiS {
if (subdomain == NULL) { if (subdomain == NULL) {
subdomain = new PetscSolverGlobalMatrix(); subdomain = new PetscSolverGlobalMatrix();
subdomain->setSymmetric(isSymmetric); subdomain->setSymmetric(isSymmetric);
subdomain->setHandleDirichletRows(false);
if (meshLevel == 0) { if (meshLevel == 0) {
subdomain->setMeshDistributor(meshDistributor, subdomain->setMeshDistributor(meshDistributor,
...@@ -134,11 +135,6 @@ namespace AMDiS { ...@@ -134,11 +135,6 @@ namespace AMDiS {
{ {
FUNCNAME("PetscSolverFeti::createDirichletData()"); FUNCNAME("PetscSolverFeti::createDirichletData()");
bool removeDirichletRows = false;
Parameters::get("parallel->feti->remove dirichlet", removeDirichletRows);
if (!removeDirichletRows)
return;
int nComponents = mat.getSize(); int nComponents = mat.getSize();
for (int i = 0; i < nComponents; i++) { for (int i = 0; i < nComponents; i++) {
DOFMatrix* dofMat = mat[i][i]; DOFMatrix* dofMat = mat[i][i];
...@@ -735,13 +731,15 @@ namespace AMDiS { ...@@ -735,13 +731,15 @@ namespace AMDiS {
InteriorBoundary &intBound = meshDistributor->getIntBoundary(); InteriorBoundary &intBound = meshDistributor->getIntBoundary();
std::set<BoundaryObject> allEdges; std::set<BoundaryObject> allEdges;
for (InteriorBoundary::iterator it(intBound.getOwn()); !it.end(); ++it) { for (InteriorBoundary::iterator it(intBound.getOwn()); !it.end(); ++it) {
if (it->rankObj.subObj == EDGE && if ((it->rankObj.subObj == FACE ||
testWirebasketEdge(it->rankObj, feSpaces[0]) && (it->rankObj.subObj == EDGE &&
testWirebasketEdge(it->rankObj, feSpaces[0]))) &&
allEdges.count(it->rankObj) == 0) { allEdges.count(it->rankObj) == 0) {
bool dirichletOnlyEdge = true; bool dirichletOnlyEdge = true;
DofContainer edgeDofs; DofContainer edgeDofs;
it->rankObj.el->getAllDofs(feSpaces[0], it->rankObj, edgeDofs); it->rankObj.el->getAllDofs(feSpaces[0], it->rankObj, edgeDofs);
for (DofContainer::iterator dit = edgeDofs.begin(); for (DofContainer::iterator dit = edgeDofs.begin();
dit != edgeDofs.end(); ++dit) { dit != edgeDofs.end(); ++dit) {
if (dirichletRows[feSpaces[0]].count(**dit) == 0) { if (dirichletRows[feSpaces[0]].count(**dit) == 0) {
...@@ -919,12 +917,10 @@ namespace AMDiS { ...@@ -919,12 +917,10 @@ namespace AMDiS {
if (creationMode == 0) { if (creationMode == 0) {