Commit 8a4eba87 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed problem with dirichlet boundary dofs in FETI-DP code.

parent a2d35454
...@@ -154,14 +154,13 @@ namespace AMDiS { ...@@ -154,14 +154,13 @@ namespace AMDiS {
MatCreateSeqAIJ(mpiCommLocal, nRankInteriorRows, nRankInteriorRows, MatCreateSeqAIJ(mpiCommLocal, nRankInteriorRows, nRankInteriorRows,
0, nnz[0][0].dnnz, 0, nnz[0][0].dnnz,
&mat[0][0]); &mat[0][0]);
MatSetOption(mat[0][0], MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} else { } else {
MatCreateAIJ(mpiCommGlobal, nRankInteriorRows, nRankInteriorRows, MatCreateAIJ(mpiCommGlobal, nRankInteriorRows, nRankInteriorRows,
nOverallInteriorRows, nOverallInteriorRows, nOverallInteriorRows, nOverallInteriorRows,
0, nnz[0][0].dnnz, 0, nnz[0][0].onnz, 0, nnz[0][0].dnnz, 0, nnz[0][0].onnz,
&mat[0][0]); &mat[0][0]);
MatSetOption(mat[0][0], MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} }
MatSetOption(mat[0][0], MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
VecCreateMPI(mpiCommGlobal, nRankInteriorRows, nOverallInteriorRows, &vecSol[0]); VecCreateMPI(mpiCommGlobal, nRankInteriorRows, nOverallInteriorRows, &vecSol[0]);
VecCreateMPI(mpiCommGlobal, nRankInteriorRows, nOverallInteriorRows, &vecRhs[0]); VecCreateMPI(mpiCommGlobal, nRankInteriorRows, nOverallInteriorRows, &vecRhs[0]);
...@@ -178,6 +177,7 @@ namespace AMDiS { ...@@ -178,6 +177,7 @@ namespace AMDiS {
nOverallCoarseRows, nOverallCoarseRows, nOverallCoarseRows, nOverallCoarseRows,
0, nnz[i + 1][i + 1].dnnz, 0, nnz[i + 1][i + 1].onnz, 0, nnz[i + 1][i + 1].dnnz, 0, nnz[i + 1][i + 1].onnz,
&mat[i + 1][i + 1]); &mat[i + 1][i + 1]);
MatSetOption(mat[i + 1][i + 1], MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
cMap->createVec(vecSol[i + 1]); cMap->createVec(vecSol[i + 1]);
cMap->createVec(vecRhs[i + 1]); cMap->createVec(vecRhs[i + 1]);
} }
......
...@@ -234,6 +234,12 @@ namespace AMDiS { ...@@ -234,6 +234,12 @@ namespace AMDiS {
} }
/// Returns whether the solver has a coarse grid.
inline bool hasCoarseSpace()
{
return (!coarseSpaceMap.empty());
}
protected: protected:
/// Prepare internal data structures. First, it create \ref uniqueCoarseMap /// Prepare internal data structures. First, it create \ref uniqueCoarseMap
/// and \ref componentIthCoarseMap . Both are used to create the correct /// and \ref componentIthCoarseMap . Both are used to create the correct
......
...@@ -108,7 +108,8 @@ namespace AMDiS { ...@@ -108,7 +108,8 @@ namespace AMDiS {
if (subdomain == NULL) { if (subdomain == NULL) {
subdomain = new PetscSolverGlobalMatrix(""); subdomain = new PetscSolverGlobalMatrix("");
subdomain->setSymmetric(isSymmetric); subdomain->setSymmetric(isSymmetric);
subdomain->setHandleDirichletRows(false); // subdomain->setHandleDirichletRows(false);
subdomain->setHandleDirichletRows(true);
if (meshLevel == 0) { if (meshLevel == 0) {
subdomain->setMeshDistributor(meshDistributor, subdomain->setMeshDistributor(meshDistributor,
...@@ -142,6 +143,8 @@ namespace AMDiS { ...@@ -142,6 +143,8 @@ namespace AMDiS {
{ {
FUNCNAME("PetscSolverFeti::createDirichletData()"); FUNCNAME("PetscSolverFeti::createDirichletData()");
return;
int nComponents = mat.getSize(); int nComponents = mat.getSize();
for (int component = 0; component < nComponents; component++) { for (int component = 0; component < nComponents; component++) {
DOFMatrix* dofMat = mat[component][component]; DOFMatrix* dofMat = mat[component][component];
......
...@@ -17,6 +17,17 @@ ...@@ -17,6 +17,17 @@
namespace AMDiS { namespace AMDiS {
PetscSolverGlobalMatrix::PetscSolverGlobalMatrix(string name)
: PetscSolver(name),
zeroStartVector(false),
printMatInfo(false)
{
Parameters::get("parallel->use zero start vector", zeroStartVector);
Parameters::get("parallel->print matrix info", printMatInfo);
}
void PetscSolverGlobalMatrix::fillPetscMatrix(Matrix<DOFMatrix*> *seqMat) void PetscSolverGlobalMatrix::fillPetscMatrix(Matrix<DOFMatrix*> *seqMat)
{ {
FUNCNAME("PetscSolverGlobalMatrix::fillPetscMatrix()"); FUNCNAME("PetscSolverGlobalMatrix::fillPetscMatrix()");
...@@ -54,7 +65,7 @@ namespace AMDiS { ...@@ -54,7 +65,7 @@ namespace AMDiS {
matAssembly(); matAssembly();
removeDirichletRows(seqMat, dofMap, getMatInterior()); removeDirichletRows(seqMat);
if (printMatInfo) { if (printMatInfo) {
MatInfo matInfo; MatInfo matInfo;
...@@ -225,6 +236,8 @@ namespace AMDiS { ...@@ -225,6 +236,8 @@ namespace AMDiS {
matAssembly(); matAssembly();
removeDirichletRows(seqMat);
// === Create solver for the non primal (thus local) variables. === // === Create solver for the non primal (thus local) variables. ===
KSPCreate(mpiCommLocal, &kspInterior); KSPCreate(mpiCommLocal, &kspInterior);
...@@ -266,7 +279,7 @@ namespace AMDiS { ...@@ -266,7 +279,7 @@ namespace AMDiS {
vecRhsAssembly(); vecRhsAssembly();
removeDirichletRows(vec, dofMap, getVecRhsInterior()); removeDirichletRows(vec);
// === For debugging allow to write the rhs vector to a file. === // === For debugging allow to write the rhs vector to a file. ===
...@@ -456,26 +469,25 @@ namespace AMDiS { ...@@ -456,26 +469,25 @@ namespace AMDiS {
} }
void PetscSolverGlobalMatrix::removeDirichletRows(Matrix<DOFMatrix*> *seqMat, void PetscSolverGlobalMatrix::removeDirichletRows(Matrix<DOFMatrix*> *seqMat)
ParallelDofMapping &dofMap,
Mat mpiMat)
{ {
FUNCNAME("PetscSolverGlobalMatrix::removeDirichletRows()"); FUNCNAME("PetscSolverGlobalMatrix::removeDirichletRows()");
if (!handleDirichletRows) if (!handleDirichletRows)
return; return;
vector<int> dRowsInterior, dRowsCoarse;
vector<int> dColsInterior, dColsCoarse;
vector<double> dValuesInterior, dValuesCoarse;
int nComponents = seqMat->getSize(); int nComponents = seqMat->getSize();
vector<int> dirichletRows;
vector<int> dirichletCols;
vector<double> dirichletValues;
for (int i = 0; i < nComponents; i++) { for (int rowComp = 0; rowComp < nComponents; rowComp++) {
bool dirichletRow = false; bool dirichletRow = false;
int dirichletMainCol = -1; int dirichletMainCol = -1;
for (int j = 0; j < nComponents; j++) { for (int colComp = 0; colComp < nComponents; colComp++) {
DOFMatrix *dofMat = (*seqMat)[i][j]; DOFMatrix *dofMat = (*seqMat)[rowComp][colComp];
if (!dofMat) if (!dofMat)
continue; continue;
...@@ -486,7 +498,7 @@ namespace AMDiS { ...@@ -486,7 +498,7 @@ namespace AMDiS {
if (bIt->second && bIt->second->isDirichlet()) { if (bIt->second && bIt->second->isDirichlet()) {
dirichletRow = true; dirichletRow = true;
if ((dynamic_cast<DirichletBC*>(bIt->second))->applyBoundaryCondition()) { if ((dynamic_cast<DirichletBC*>(bIt->second))->applyBoundaryCondition()) {
dirichletMainCol = j; dirichletMainCol = colComp;
break; break;
} }
} }
...@@ -496,65 +508,152 @@ namespace AMDiS { ...@@ -496,65 +508,152 @@ namespace AMDiS {
if (!dirichletRow) if (!dirichletRow)
continue; continue;
DOFMatrix *dofMat = (*seqMat)[i][dirichletMainCol]; DOFMatrix *dofMat = (*seqMat)[rowComp][dirichletMainCol];
const FiniteElemSpace *feSpace = dofMat->getRowFeSpace();
TEST_EXIT(dofMat->getRowFeSpace() == dofMat->getColFeSpace()) TEST_EXIT(dofMat->getRowFeSpace() == dofMat->getColFeSpace())
("I have to think about this scenario! Really possible?\n"); ("I have to think about this scenario! Really possible?\n");
std::set<DegreeOfFreedom> &dRows = dofMat->getDirichletRows(); std::set<DegreeOfFreedom> &dRows = dofMat->getDirichletRows();
for (std::set<DegreeOfFreedom>::iterator dofIt = dRows.begin(); for (std::set<DegreeOfFreedom>::iterator dofIt = dRows.begin();
dofIt != dRows.end(); ++dofIt) { dofIt != dRows.end(); ++dofIt) {
MultiIndex multiIndex; if (hasCoarseSpace()) {
if (dofMap[feSpace].find(*dofIt, multiIndex) && bool isRowCoarse = isCoarseSpace(rowComp, *dofIt);
dofMap[feSpace].isRankDof(*dofIt)) { bool isColCoarse = isCoarseSpace(dirichletMainCol, *dofIt);
int rowIndex = dofMap.getMatIndex(i, multiIndex.global); TEST_EXIT(isRowCoarse == isColCoarse)
int colIndex = dofMap.getMatIndex(dirichletMainCol, multiIndex.global); ("Really possible? So reimplement AMDiS from the scratch!\n");
dirichletRows.push_back(rowIndex);
dirichletCols.push_back(colIndex); if (isRowCoarse) {
dirichletValues.push_back(1.0); ParallelDofMapping *rowCoarseSpace = coarseSpaceMap[rowComp];
ParallelDofMapping *colCoarseSpace = coarseSpaceMap[dirichletMainCol];
MultiIndex multiIndex;
if ((*rowCoarseSpace)[rowComp].find(*dofIt, multiIndex) &&
(*rowCoarseSpace)[rowComp].isRankDof(*dofIt)) {
int rowIndex = rowCoarseSpace->getLocalMatIndex(rowComp, *dofIt);
int colIndex = colCoarseSpace->getLocalMatIndex(dirichletMainCol, *dofIt);
dRowsCoarse.push_back(rowIndex);
dColsCoarse.push_back(colIndex);
dValuesCoarse.push_back(1.0);
}
} else {
MultiIndex multiIndex;
if ((*interiorMap)[rowComp].find(*dofIt, multiIndex) &&
(*interiorMap)[rowComp].isRankDof(*dofIt)) {
int rowIndex = interiorMap->getLocalMatIndex(rowComp, *dofIt);
int colIndex = interiorMap->getLocalMatIndex(dirichletMainCol, *dofIt);
dRowsInterior.push_back(rowIndex);
dColsInterior.push_back(colIndex);
dValuesInterior.push_back(1.0);
}
}
} else {
MultiIndex multiIndex;
if ((*interiorMap)[rowComp].find(*dofIt, multiIndex) &&
(*interiorMap)[rowComp].isRankDof(*dofIt)) {
int rowIndex = interiorMap->getMatIndex(rowComp, multiIndex.global);
int colIndex = interiorMap->getMatIndex(dirichletMainCol, multiIndex.global);
dRowsInterior.push_back(rowIndex);
dColsInterior.push_back(colIndex);
dValuesInterior.push_back(1.0);
}
} }
} }
} }
MatZeroRows(mpiMat, dirichletRows.size(), &(dirichletRows[0]), 0.0, {
PETSC_NULL, PETSC_NULL); Mat mpiMat = getMatInterior();
MatZeroRows(mpiMat, dRowsInterior.size(), &(dRowsInterior[0]), 0.0,
PETSC_NULL, PETSC_NULL);
for (int i = 0; i < static_cast<int>(dRowsInterior.size()); i++)
MatSetValue(mpiMat, dRowsInterior[i], dColsInterior[i],
dValuesInterior[i], INSERT_VALUES);
MatAssemblyBegin(mpiMat, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(mpiMat, MAT_FINAL_ASSEMBLY);
}
for (int i = 0; i < static_cast<int>(dirichletRows.size()); i++) if (hasCoarseSpace()) {
MatSetValue(mpiMat, dirichletRows[i], dirichletCols[i], dirichletValues[i], INSERT_VALUES); Mat mpiMat = getMatCoarse();
MatZeroRows(mpiMat, dRowsCoarse.size(), &(dRowsCoarse[0]), 0.0,
MatAssemblyBegin(mpiMat, MAT_FINAL_ASSEMBLY); PETSC_NULL, PETSC_NULL);
MatAssemblyEnd(mpiMat, MAT_FINAL_ASSEMBLY);
for (int i = 0; i < static_cast<int>(dRowsCoarse.size()); i++)
MatSetValue(mpiMat, dRowsCoarse[i], dColsCoarse[i],
dValuesCoarse[i], INSERT_VALUES);
MatAssemblyBegin(mpiMat, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(mpiMat, MAT_FINAL_ASSEMBLY);
}
} }
void PetscSolverGlobalMatrix::removeDirichletRows(SystemVector *seqVec, void PetscSolverGlobalMatrix::removeDirichletRows(SystemVector *seqVec)
ParallelDofMapping &dofMap,
Vec mpiVec)
{ {
FUNCNAME("PetscSolverGlobalMatrix::removeDirichletRows()"); FUNCNAME("PetscSolverGlobalMatrix::removeDirichletRows()");
if (!handleDirichletRows) if (!handleDirichletRows)
return; return;
int nComponents = seqVec->getSize(); int cInterior = 0;
int cCoarse = 0;
for (int i = 0; i < nComponents; i++) {
DOFVector<double> *dofVec = seqVec->getDOFVector(i);
const FiniteElemSpace *feSpace = dofVec->getFeSpace();
int nComponents = seqVec->getSize();
for (int component = 0; component < nComponents; component++) {
DOFVector<double> *dofVec = seqVec->getDOFVector(component);
map<DegreeOfFreedom, double>& dValues = dofVec->getDirichletValues(); map<DegreeOfFreedom, double>& dValues = dofVec->getDirichletValues();
for (map<DegreeOfFreedom, double>::iterator dofIt = dValues.begin(); for (map<DegreeOfFreedom, double>::iterator dofIt = dValues.begin();
dofIt != dValues.end(); ++dofIt) { dofIt != dValues.end(); ++dofIt) {
MultiIndex multiIndex; if (hasCoarseSpace()) {
if (dofMap[feSpace].find(dofIt->first, multiIndex) && if (isCoarseSpace(component, dofIt->first)) {
dofMap[feSpace].isRankDof(dofIt->first)) ParallelDofMapping *rowCoarseSpace = coarseSpaceMap[component];
VecSetValue(mpiVec, dofMap.getMatIndex(i, multiIndex.global), MultiIndex multiIndex;
dofIt->second, INSERT_VALUES); if ((*rowCoarseSpace)[component].find(dofIt->first, multiIndex) &&
(*rowCoarseSpace)[component].isRankDof(dofIt->first)) {
if (dofIt->second > 0.5)
cCoarse++;
VecSetValue(getVecRhsCoarse(),
rowCoarseSpace->getMatIndex(component, multiIndex.global),
dofIt->second, INSERT_VALUES);
}
} else {
MultiIndex multiIndex;
if ((*interiorMap)[component].find(dofIt->first, multiIndex) &&
(*interiorMap)[component].isRankDof(dofIt->first)) {
if (dofIt->second > 0.5)
cInterior++;
VecSetValue(getVecRhsInterior(),
interiorMap->getLocalMatIndex(component, dofIt->first),
dofIt->second, INSERT_VALUES);
}
}
} else {
MultiIndex multiIndex;
if ((*interiorMap)[component].find(dofIt->first, multiIndex) &&
(*interiorMap)[component].isRankDof(dofIt->first)) {
if (dofIt->second > 0.5)
cInterior++;
VecSetValue(getVecRhsInterior(),
interiorMap->getMatIndex(component, multiIndex.global),
dofIt->second, INSERT_VALUES);
}
}
} }
} }
VecAssemblyBegin(mpiVec); VecAssemblyBegin(getVecRhsInterior());
VecAssemblyEnd(mpiVec); VecAssemblyEnd(getVecRhsInterior());
if (hasCoarseSpace()) {
VecAssemblyBegin(getVecRhsCoarse());
VecAssemblyEnd(getVecRhsCoarse());
}
mpi::globalAdd(cInterior);
mpi::globalAdd(cCoarse);
MSG("WORKED ON DIRICHLET DOFS: %d %d\n", cInterior, cCoarse);
} }
......
...@@ -39,14 +39,7 @@ namespace AMDiS { ...@@ -39,14 +39,7 @@ namespace AMDiS {
class PetscSolverGlobalMatrix : public PetscSolver class PetscSolverGlobalMatrix : public PetscSolver
{ {
public: public:
PetscSolverGlobalMatrix(string name) PetscSolverGlobalMatrix(string name);
: PetscSolver(name),
zeroStartVector(false),
printMatInfo(false)
{
Parameters::get("parallel->use zero start vector", zeroStartVector);
Parameters::get("parallel->print matrix info", printMatInfo);
}
void fillPetscMatrix(Matrix<DOFMatrix*> *mat); void fillPetscMatrix(Matrix<DOFMatrix*> *mat);
...@@ -63,13 +56,9 @@ namespace AMDiS { ...@@ -63,13 +56,9 @@ namespace AMDiS {
void destroyVectorData(); void destroyVectorData();
protected: protected:
void removeDirichletRows(Matrix<DOFMatrix*> *seqMat, void removeDirichletRows(Matrix<DOFMatrix*> *seqMat);
ParallelDofMapping &dofMap,
Mat mpiMat);
void removeDirichletRows(SystemVector *seqVec, void removeDirichletRows(SystemVector *seqVec);
ParallelDofMapping &dofMap,
Vec mpiVec);
/// Reads field split information and creats a splitting based on /// Reads field split information and creats a splitting based on
/// component numbers. /// component numbers.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment