Skip to content

GitLab

Commit 2e22f957 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

PETSc global block solver should now work correctly.

parent ac22232d
Hide whitespace changes
Inline Side-by-side
Showing with 144 additions and 95 deletions
AMDiS/src/parallel/MeshDistributor.cc
View file @ 2e22f957
......@@ -78,7 +78,7 @@ namespace AMDiS {
partitioner(NULL),
nRankDofs(0),
nOverallDofs(0),
rstart(0),
rStartDofs(0),
deserialized(false),
writeSerializationFile(false),
repartitioningAllowed(false),
......@@ -1837,13 +1837,13 @@ namespace AMDiS {
// Get displacment for global rank DOF ordering and global DOF number.
nRankDofs = rankDofs.size();
mpi::getDofNumbering(mpiComm, nRankDofs, rstart, nOverallDofs);
mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
// Stores for all rank owned DOFs a new global index.
DofIndexMap rankDofsNewGlobalIndex;
for (int i = 0; i < nRankDofs; i++)
rankDofsNewGlobalIndex[rankDofs[i]] = i + rstart;
rankDofsNewGlobalIndex[rankDofs[i]] = i + rStartDofs;
// === Send and receive new DOF indices. ===
......@@ -1906,7 +1906,7 @@ namespace AMDiS {
MSG("------------- Debug information -------------\n");
MSG("nRankDofs = %d\n", nRankDofs);
MSG("nOverallDofs = %d\n", nOverallDofs);
MSG("rstart %d\n", rstart);
MSG("rStartDofs %d\n", rStartDofs);
stringstream oss;
oss << debugOutputDir << "elementIndex-" << mpiRank << ".vtu";
......@@ -2170,7 +2170,7 @@ namespace AMDiS {
serialize(out, periodicDof);
serialize(out, periodicDofAssociations);
SerUtil::serialize(out, rstart);
SerUtil::serialize(out, rStartDofs);
SerUtil::serialize(out, macroElementNeighbours);
int nSize = allMacroElements.size();
......@@ -2231,7 +2231,7 @@ namespace AMDiS {
deserialize(in, periodicDof);
deserialize(in, periodicDofAssociations);
SerUtil::deserialize(in, rstart);
SerUtil::deserialize(in, rStartDofs);
SerUtil::deserialize(in, macroElementNeighbours);
int nSize = 0;
......
AMDiS/src/parallel/MeshDistributor.h
View file @ 2e22f957
......@@ -136,6 +136,12 @@ namespace AMDiS {
return nRankDofs;
}
/// Returns \ref rStartDofs, the first global DOF index owned by rank.
inline int getStartDofs()
{
return rStartDofs;
}
/// Returns \ref nOverallDofs, the global number of DOFs.
inline int getNumberOverallDofs()
{
......@@ -225,11 +231,6 @@ namespace AMDiS {
return lastMeshChangeIndex;
}
inline int getRstart()
{
return rstart;
}
inline int getMpiRank()
{
return mpiRank;
......@@ -521,6 +522,9 @@ namespace AMDiS {
/// Number of DOFs in the rank mesh.
int nRankDofs;
/// Is the index of the first global DOF index, which is owned by the rank.
int rStartDofs;
/// Number of DOFs in the whole domain.
int nOverallDofs;
......@@ -604,10 +608,6 @@ namespace AMDiS {
/// repartitioned.
vector<DOFVector<double>*> interchangeVectors;
/// Is the index of the first row of the linear system, which is owned by
/// the rank.
int rstart;
/** \brief
* If the problem definition has been read from a serialization file, this
* variable is true, otherwise it is false. This variable is used to stop the
......
AMDiS/src/parallel/PetscProblemStat.cc
View file @ 2e22f957
......@@ -25,7 +25,8 @@ namespace AMDiS {
PetscProblemStat::PetscProblemStat(string nameStr,
ProblemIterationInterface *problemIteration)
: ParallelProblemStatBase(nameStr, problemIteration)
: ParallelProblemStatBase(nameStr, problemIteration),
deleteSolver(true)
{
FUNCNAME("PetscProblemStat::PetscProblemStat()");
......@@ -54,6 +55,14 @@ namespace AMDiS {
}
PetscProblemStat::PetscProblemStat(string nameStr,
PetscSolver *solver)
: ParallelProblemStatBase(nameStr, NULL),
petscSolver(solver),
deleteSolver(false)
{}
void PetscProblemStat::initialize(Flag initFlag,
ProblemStatSeq* adoptProblem,
Flag adoptFlag)
......
AMDiS/src/parallel/PetscProblemStat.h
View file @ 2e22f957
......@@ -40,9 +40,13 @@ namespace AMDiS {
PetscProblemStat(std::string nameStr,
ProblemIterationInterface *problemIteration = NULL);
PetscProblemStat(std::string nameStr,
PetscSolver *solver);
~PetscProblemStat()
{
delete petscSolver;
if (deleteSolver)
delete petscSolver;
}
void initialize(Flag initFlag,
......@@ -55,6 +59,8 @@ namespace AMDiS {
protected:
PetscSolver *petscSolver;
bool deleteSolver;
};
typedef PetscProblemStat ParallelProblemStat;
......
AMDiS/src/parallel/PetscSolver.cc
View file @ 2e22f957
......@@ -40,4 +40,35 @@ namespace AMDiS {
}
}
void PetscSolver::copyVec(Vec& originVec, Vec& destVec,
vector<int>& originIndex, vector<int>& destIndex)
{
FUNCNAME("PetscSolver::copyVec()");
IS originIs, destIs;
ISCreateGeneral(PETSC_COMM_WORLD,
originIndex.size(),
&(originIndex[0]),
PETSC_USE_POINTER,
&originIs);
ISCreateGeneral(PETSC_COMM_WORLD,
destIndex.size(),
&(destIndex[0]),
PETSC_USE_POINTER,
&destIs);
VecScatter scatter;
VecScatterCreate(originVec, originIs, destVec, destIs, &scatter);
VecScatterBegin(scatter, originVec, destVec,
INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(scatter, originVec, destVec,
INSERT_VALUES, SCATTER_FORWARD);
ISDestroy(&originIs);
ISDestroy(&destIs);
VecScatterDestroy(&scatter);
}
}
AMDiS/src/parallel/PetscSolver.h
View file @ 2e22f957
......@@ -90,6 +90,20 @@ namespace AMDiS {
bool iterationCounter = true,
bool residual = true);
/** \brief
* Copies between to PETSc vectors by using different index sets for the
* origin and the destination vectors.
*
* \param[in] originVec The PETSc vector from which we copy from.
* \param[out] destVec The PETSc vector we copy too.
* \param[in] originIndex Set of global indices referring to the
* origin vector.
* \param[in] destIndex Set of global indices referring to the
* destination vector.
*/
void copyVec(Vec& originVec, Vec& destVec,
vector<int>& originIndex, vector<int>& destIndex);
protected:
MeshDistributor *meshDistributor;
......
AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.cc
View file @ 2e22f957
......@@ -25,22 +25,14 @@ namespace AMDiS {
double wtime = MPI::Wtime();
nComponents = mat->getNumRows();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
// === Create PETSc vector (solution and a temporary vector). ===
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscSolVec);
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscTmpVec);
int nRankRows = meshDistributor->getNumberRankDofs();
int nOverallRows = meshDistributor->getNumberOverallDofs();
#if (DEBUG != 0)
MSG("Fill petsc matrix 1 needed %.5f seconds\n", MPI::Wtime() - wtime);
#endif
nestMat = new Mat*[nComponents];
for (int i = 0; i < nComponents; i++)
nestMat[i] = new Mat[nComponents];
nestMat.resize(nComponents * nComponents);
// === Transfer values from DOF matrices to the PETSc matrix. ===
......@@ -51,16 +43,17 @@ namespace AMDiS {
nRankRows, nRankRows,
nOverallRows, nOverallRows,
30, PETSC_NULL, 30, PETSC_NULL,
&(nestMat[i][j]));
setDofMatrix(nestMat[i][j], (*mat)[i][j]);
&(nestMat[i * nComponents + j]));
setDofMatrix(nestMat[i * nComponents + j], (*mat)[i][j]);
MatAssemblyBegin(nestMat[i * nComponents + j], MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(nestMat[i * nComponents + j], MAT_FINAL_ASSEMBLY);
} else {
nestMat[i][j] = PETSC_NULL;
nestMat[i * nComponents + j] = PETSC_NULL;
}
MatCreateNest(PETSC_COMM_WORLD,
nComponents, PETSC_NULL, nComponents, PETSC_NULL,
&(nestMat[0][0]), &petscMatrix);
&(nestMat[0]), &petscMatrix);
#if (DEBUG != 0)
MSG("Fill petsc matrix 2 needed %.5f seconds\n", MPI::Wtime() - wtime);
......@@ -71,12 +64,11 @@ namespace AMDiS {
// === Init PETSc solver. ===
KSPCreate(PETSC_COMM_WORLD, &solver);
KSPGetPC(solver, &pc);
KSPSetOperators(solver, petscMatrix, petscMatrix, SAME_NONZERO_PATTERN);
KSPSetTolerances(solver, 0.0, 1e-8, PETSC_DEFAULT, PETSC_DEFAULT);
KSPSetType(solver, KSPBCGS);
KSPSetFromOptions(solver);
PCSetFromOptions(pc);
KSPGetPC(solver, &pc);
setBlockPreconditioner(pc);
MSG("Fill petsc matrix needed %.5f seconds\n", MPI::Wtime() - wtime);
}
......@@ -89,15 +81,22 @@ namespace AMDiS {
TEST_EXIT_DBG(vec)("NO DOF vector defined!\n");
nComponents = vec->getSize();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
int nRankRows = meshDistributor->getNumberRankDofs();
int nOverallRows = meshDistributor->getNumberOverallDofs();
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &petscRhsVec);
nestVec.resize(nComponents);
// === Transfer values from DOF vector to the PETSc vector. ===
for (int i = 0; i < nComponents; i++) {
VecCreateMPI(PETSC_COMM_WORLD, nRankRows, nOverallRows, &(nestVec[i]));
for (int i = 0; i < nComponents; i++)
setDofVector(petscRhsVec, vec->getDOFVector(i), nComponents, i);
setDofVector(nestVec[i], vec->getDOFVector(i));
VecAssemblyBegin(nestVec[i]);
VecAssemblyEnd(nestVec[i]);
}
VecCreateNest(PETSC_COMM_WORLD, nComponents, PETSC_NULL,
&(nestVec[0]), &petscRhsVec);
VecAssemblyBegin(petscRhsVec);
VecAssemblyEnd(petscRhsVec);
......@@ -110,21 +109,24 @@ namespace AMDiS {
FUNCNAME("PetscSolverGlobalBlockMatrix::solvePetscMatrix()");
// PETSc.
KSPSolve(solver, petscRhsVec, petscSolVec);
KSPSolve(solver, petscRhsVec, petscRhsVec);
// === Transfere values from PETSc's solution vectors to the DOF vectors. ===
int nRankDofs = meshDistributor->getNumberRankDofs();
PetscScalar *vecPointer;
VecGetArray(petscSolVec, &vecPointer);
for (int i = 0; i < nComponents; i++) {
DOFVector<double> &dofvec = *(vec.getDOFVector(i));
Vec tmp;
VecNestGetSubVec(petscRhsVec, i, &tmp);
int nRankDofs = meshDistributor->getNumberRankDofs();
PetscScalar *vecPointer;
VecGetArray(tmp, &vecPointer);
for (int j = 0; j < nRankDofs; j++)
dofvec[meshDistributor->mapLocalToDofIndex(j)] =
vecPointer[i * nComponents + j];
}
dofvec[meshDistributor->mapLocalToDofIndex(j)] = vecPointer[j];
VecRestoreArray(petscSolVec, &vecPointer);
VecRestoreArray(tmp, &vecPointer);
}
// === Synchronize DOFs at common DOFs, i.e., DOFs that correspond to ===
......@@ -132,12 +134,10 @@ namespace AMDiS {
meshDistributor->synchVector(vec);
// Print iteration counter and residual norm of the solution.
printSolutionInfo(adaptInfo);
// === Destroy PETSc's variables. ===
VecDestroy(&petscRhsVec);
for (int i = 0; i < nComponents; i++)
VecDestroy(&(nestVec[i]));
}
......@@ -145,20 +145,12 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverGlobalBlockMatrix::destroyMatrixData()");
for (unsigned int i = 0; i < nestMat.size(); i++)
if (nestMat[i] != PETSC_NULL)
MatDestroy(&(nestMat[i]));
MatDestroy(&petscMatrix);
KSPDestroy(&solver);
VecDestroy(&petscSolVec);
VecDestroy(&petscTmpVec);
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++) {
if (nestMat[i][j] != PETSC_NULL)
MatDestroy(&(nestMat[i][j]));
}
delete[] nestMat[i];
}
delete[] nestMat;
}
......@@ -209,32 +201,22 @@ namespace AMDiS {
cols.push_back(colIndex);
values.push_back(value(*icursor));
}
MatSetValues(petscMatrix, 1, &rowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES);
MatSetValues(petscMat, 1, &rowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES);
}
}
void PetscSolverGlobalBlockMatrix::setDofVector(Vec& petscVec,
DOFVector<double>* vec,
int nComponents,
int row,
bool rankOnly)
DOFVector<double>* vec)
{
FUNCNAME("PetscSolverGlobalBlockMatrix::setDofVector()");
// Traverse all used DOFs in the dof vector.
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
for (dofIt.reset(); !dofIt.end(); ++dofIt) {
if (rankOnly && !meshDistributor->getIsRankDof(dofIt.getDOFIndex()))
continue;
// Calculate global row index of the DOF.
DegreeOfFreedom globalRowDof =
meshDistributor->mapLocalToGlobal(dofIt.getDOFIndex());
// Calculate PETSc index of the row DOF.
int index = nComponents * row + globalRowDof;
int index = meshDistributor->mapLocalToGlobal(dofIt.getDOFIndex());
double value = *dofIt;
VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
......
AMDiS/src/parallel/PetscSolverGlobalBlockMatrix.h
View file @ 2e22f957
......@@ -52,11 +52,18 @@ namespace AMDiS {
void setDofMatrix(Mat& petscMat, DOFMatrix* mat);
/// Takes a DOF vector and sends its values to a given PETSc vector.
void setDofVector(Vec& petscVec, DOFVector<double>* vec,
int nComponents, int row, bool rankOnly = false);
void setDofVector(Vec& petscVec, DOFVector<double>* vec);
virtual void setBlockPreconditioner(PC &pc)
{
PCSetFromOptions(pc);
}
protected:
Mat **nestMat;
vector<Mat> nestMat;
vector<Vec> nestVec;
int nComponents;
};
......
AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
View file @ 2e22f957
......@@ -66,9 +66,9 @@ namespace AMDiS {
#if (DEBUG != 0)
int a, b;
MatGetOwnershipRange(petscMatrix, &a, &b);
TEST_EXIT(a == meshDistributor->getRstart() * nComponents)
TEST_EXIT(a == meshDistributor->getStartDofs() * nComponents)
("Wrong matrix ownership range!\n");
TEST_EXIT(b == meshDistributor->getRstart() * nComponents + nRankRows)
TEST_EXIT(b == meshDistributor->getStartDofs() * nComponents + nRankRows)
("Wrong matrix ownership range!\n");
#endif
......@@ -552,7 +552,7 @@ namespace AMDiS {
// This is the local row index of the local PETSc matrix.
int localPetscRowIdx =
petscRowIdx - meshDistributor->getRstart() * nComponents;
petscRowIdx - meshDistributor->getStartDofs() * nComponents;
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows)
("Should not happen! \n Debug info: localRowIdx = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d nCompontens = %d nRankRows = %d\n",
......@@ -560,7 +560,7 @@ namespace AMDiS {
meshDistributor->mapLocalToGlobal(*cursor),
petscRowIdx,
localPetscRowIdx,
meshDistributor->getRstart(),
meshDistributor->getStartDofs(),
nComponents,
nRankRows);
......@@ -574,8 +574,8 @@ namespace AMDiS {
if (value(*icursor) != 0.0 || petscRowIdx == petscColIdx) {
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the d_nnz values for this row, otherwise the o_nnz value.
if (petscColIdx >= meshDistributor->getRstart() * nComponents &&
petscColIdx < meshDistributor->getRstart() * nComponents + nRankRows)
if (petscColIdx >= meshDistributor->getStartDofs() * nComponents &&
petscColIdx < meshDistributor->getStartDofs() * nComponents + nRankRows)
d_nnz[localPetscRowIdx]++;
else
o_nnz[localPetscRowIdx]++;
......@@ -626,14 +626,14 @@ namespace AMDiS {
int r = it->second[i].first;
int c = it->second[i].second;
int localRowIdx = r - meshDistributor->getRstart() * nComponents;
int localRowIdx = r - meshDistributor->getStartDofs() * nComponents;
TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows)
("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n",
r, localRowIdx, nRankRows, it->first);
if (c < meshDistributor->getRstart() * nComponents ||
c >= meshDistributor->getRstart() * nComponents + nRankRows)
if (c < meshDistributor->getStartDofs() * nComponents ||
c >= meshDistributor->getStartDofs() * nComponents + nRankRows)
o_nnz[localRowIdx]++;
else
d_nnz[localRowIdx]++;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment