Skip to content
GitLab
Commit 2bbf10f7 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed some problems with nnz computation with coarse grid, still not finished.

parent 53c5f965
Hide whitespace changes
Inline Side-by-side
AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
View file @ 2bbf10f7
......@@ -51,7 +51,9 @@ namespace AMDiS {
MatCreateAIJ(mpiCommGlobal, nRankRows, nRankRows,
nOverallRows, nOverallRows,
0, matInteriorDiagNnz, 0, matInteriorOffdiagNnz, &matIntInt);
0, nnzInterior.dnnz,
0, nnzInterior.onnz,
&matIntInt);
MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
......@@ -132,28 +134,40 @@ namespace AMDiS {
bool localMatrix = (subdomainLevel == 0);
if (checkMeshChange(mat, localMatrix)) {
createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap,
matInteriorDiagNnz,
matInteriorOffdiagNnz,
localMatrix);
if (coarseSpaceMap)
createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap,
*interiorMap,
nnzInterior,
localMatrix);
if (coarseSpaceMap) {
createPetscNnzStructureWithCoarseSpace(mat,
*coarseSpaceMap,
*coarseSpaceMap,
nnzCoarse);
createPetscNnzStructureWithCoarseSpace(mat,
*coarseSpaceMap,
matCoarseDiagNnz,
matCoarseOffdiagNnz);
*coarseSpaceMap,
*interiorMap,
nnzCoarseInt);
createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap,
*coarseSpaceMap,
nnzIntCoarse);
}
}
if (localMatrix) {
MatCreateSeqAIJ(mpiCommLocal, nRowsRankInterior, nRowsRankInterior,
0, matInteriorDiagNnz, &matIntInt);
0, nnzInterior.dnnz,
&matIntInt);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} else {
MatCreateAIJ(mpiCommLocal,
nRowsRankInterior, nRowsRankInterior,
nRowsOverallInterior, nRowsOverallInterior,
0, matInteriorDiagNnz, 0, matInteriorOffdiagNnz,
0, nnzInterior.dnnz, 0, nnzInterior.onnz,
&matIntInt);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
if (coarseSpaceMap) {
......@@ -163,13 +177,16 @@ namespace AMDiS {
MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankCoarse,
nRowsOverallCoarse, nRowsOverallCoarse,
0, matCoarseDiagNnz, 0, matCoarseOffdiagNnz,
0, nnzCoarse.dnnz, 0, nnzCoarse.onnz,
// 100, PETSC_NULL, 100, PETSC_NULL,
&matCoarseCoarse);
// MatSetOption(matCoarseCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankInterior,
nRowsOverallCoarse, nGlobalOverallInterior,
100, PETSC_NULL, 100, PETSC_NULL,
// 0, nnzCoarseInt.dnnz, 0, nnzCoarseInt.onnz,
&matCoarseInt);
MatSetOption(matCoarseInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
......@@ -177,7 +194,7 @@ namespace AMDiS {
nRowsRankInterior, nRowsRankCoarse,
nGlobalOverallInterior, nRowsOverallCoarse,
100, PETSC_NULL, 100, PETSC_NULL,
// 0, matInteriorDiagNnz, 0, matCoarseOffdiagNnz,
// 0, nnzIntCoarse.dnnz, 0, nnzIntCoarse.onnz,
&matIntCoarse);
MatSetOption(matIntCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
......@@ -259,14 +276,19 @@ namespace AMDiS {
if (colsOther.size()) {
if (subdomainLevel == 0) {
for (unsigned int k = 0; k < colsOther.size(); k++)
for (unsigned int k = 0; k < colsOther.size(); k++) {
int local = interiorMap->getMatIndex(j, colsOther[k]) - interiorMap->getStartDofs();
if (rowIndex == 0 && local == 9297)
MSG("FOUND: %d %d %d\n", colsOther[k], j, interiorMap->getMatIndex(j, colsOther[k]));
colsOther[k] = interiorMap->getMatIndex(j, colsOther[k]);
}
} else {
for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] =
interiorMap->getMatIndex(j, colsOther[k]) + rStartInterior;
}
MatSetValues(matCoarseInt, 1, &rowIndex, colsOther.size(),
&(colsOther[0]), &(valuesOther[0]), ADD_VALUES);
}
......@@ -318,6 +340,8 @@ namespace AMDiS {
MatAssemblyEnd(matCoarseInt, MAT_FINAL_ASSEMBLY);
}
AMDiS::finalize();
exit(0);
// === Remove Dirichlet BC DOFs. ===
......@@ -577,30 +601,18 @@ namespace AMDiS {
static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
mpiCommGlobal.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
if (!matInteriorDiagNnz || recvAllValues != 0 || alwaysCreateNnzStructure) {
if (!nnzInterior.dnnz || recvAllValues != 0 || alwaysCreateNnzStructure) {
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
interiorMap->setComputeMatIndex(true, !localMatrix);
interiorMap->update(feSpaces);
if (matInteriorDiagNnz) {
delete [] matInteriorDiagNnz;
matInteriorDiagNnz = NULL;
}
if (matInteriorOffdiagNnz) {
delete [] matInteriorOffdiagNnz;
matInteriorOffdiagNnz = NULL;
}
nnzInterior.clear();
if (matCoarseDiagNnz) {
delete [] matCoarseDiagNnz;
matCoarseDiagNnz = NULL;
}
if (matCoarseOffdiagNnz) {
delete [] matCoarseOffdiagNnz;
matCoarseOffdiagNnz = NULL;
if (coarseSpaceMap) {
nnzCoarse.clear();
nnzCoarseInt.clear();
nnzIntCoarse.clear();
}
updateSubdomainData();
......@@ -1034,18 +1046,11 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()");
TEST_EXIT_DBG(!matInteriorDiagNnz)("There is something wrong!\n");
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
int nRankRows = interiorMap->getRankDofs();
int rankStartIndex = interiorMap->getStartDofs();
matInteriorDiagNnz = new int[nRankRows];
matInteriorOffdiagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++) {
matInteriorDiagNnz[i] = 0;
matInteriorOffdiagNnz[i] = 0;
}
nnzInterior.create(nRankRows, nRankRows);
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits;
......@@ -1143,9 +1148,9 @@ namespace AMDiS {
// otherwise the matInteriorDiagNnz value.
if (petscColIdx >= rankStartIndex &&
petscColIdx < rankStartIndex + nRankRows)
matInteriorDiagNnz[localPetscRowIdx]++;
nnzInterior.dnnz[localPetscRowIdx]++;
else
matInteriorOffdiagNnz[localPetscRowIdx]++;
nnzInterior.onnz[localPetscRowIdx]++;
}
}
} else {
......@@ -1202,9 +1207,9 @@ namespace AMDiS {
r, localRowIdx, nRankRows, it->first);
if (c < rankStartIndex || c >= rankStartIndex + nRankRows)
matInteriorOffdiagNnz[localRowIdx]++;
nnzInterior.onnz[localRowIdx]++;
else
matInteriorDiagNnz[localRowIdx]++;
nnzInterior.dnnz[localRowIdx]++;
}
}
}
......@@ -1217,33 +1222,26 @@ namespace AMDiS {
if (nRankRows < 100)
for (int i = 0; i < nRankRows; i++)
matInteriorDiagNnz[i] = std::min(matInteriorDiagNnz[i], nRankRows);
nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankRows);
}
void PetscSolverGlobalMatrix::createPetscNnzStructureWithCoarseSpace(Matrix<DOFMatrix*> *mat,
ParallelDofMapping &dofMap,
int*& diagNnz,
int*& offdiagNnz,
ParallelDofMapping &rowDofMap,
ParallelDofMapping &colDofMap,
NnzStructure &nnz,
bool localMatrix)
{
FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()");
TEST_EXIT_DBG(!diagNnz)("There is something wrong!\n");
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
int nRankRows = dofMap.getRankDofs();
int rankStartIndex = dofMap.getStartDofs();
int nRankRows = rowDofMap.getRankDofs();
int rankStartRowIndex = rowDofMap.getStartDofs();
diagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++)
diagNnz[i] = 0;
int nRankCols = colDofMap.getRankDofs();
int rankStartColIndex = colDofMap.getStartDofs();
if (localMatrix == false) {
offdiagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++)
offdiagNnz[i] = 0;
}
nnz.create(nRankRows, (!localMatrix ? nRankRows : -1));
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits;
......@@ -1308,21 +1306,21 @@ namespace AMDiS {
for (cursor_type cursor = begin<row>(bmat),
cend = end<row>(bmat); cursor != cend; ++cursor) {
if (dofMap[feSpaces[i]].find(*cursor, rowDofIndex) == false)
if (rowDofMap[feSpaces[i]].find(*cursor, rowDofIndex) == false)
continue;
// The corresponding global matrix row index of the current row DOF.
int petscRowIdx = 0;
if (localMatrix) {
petscRowIdx = dofMap.getLocalMatIndex(i, *cursor);
petscRowIdx = rowDofMap.getLocalMatIndex(i, *cursor);
} else {
if (dofMap.isMatIndexFromGlobal())
petscRowIdx = dofMap.getMatIndex(i, rowDofIndex.global);
if (rowDofMap.isMatIndexFromGlobal())
petscRowIdx = rowDofMap.getMatIndex(i, rowDofIndex.global);
else
petscRowIdx = dofMap.getMatIndex(i, *cursor);
petscRowIdx = rowDofMap.getMatIndex(i, *cursor);
}
if (localMatrix || dofMap[feSpaces[i]].isRankDof(*cursor)) {
if (localMatrix || rowDofMap[feSpaces[i]].isRankDof(*cursor)) {
// === The current row DOF is a rank DOF, so create the ===
// === corresponding nnz values directly on rank's nnz data. ===
......@@ -1330,50 +1328,47 @@ namespace AMDiS {
// This is the local row index of the local PETSc matrix.
int localPetscRowIdx = petscRowIdx;
if (localMatrix == false) {
localPetscRowIdx -= rankStartIndex;
if (localMatrix == false)
localPetscRowIdx -= rankStartRowIndex;
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows)
("Should not happen! \n Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d\n",
*cursor,
dofMap[feSpaces[i]][*cursor].global,
petscRowIdx,
localPetscRowIdx,
rankStartIndex,
i,
nComponents,
nRankRows);
}
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows)
("Should not happen! \n Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d\n",
*cursor,
rowDofMap[feSpaces[i]][*cursor].global,
petscRowIdx,
localPetscRowIdx,
rankStartRowIndex,
i,
nComponents,
nRankRows);
if (localMatrix) {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor)
if (dofMap[feSpaces[j]].isSet(col(*icursor)))
diagNnz[localPetscRowIdx]++;
if (colDofMap[feSpaces[j]].isSet(col(*icursor)))
nnz.dnnz[localPetscRowIdx]++;
} else {
MultiIndex colDofIndex;
// Traverse all non zero entries in this row.
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (dofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
if (colDofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
continue;
int petscColIdx = (dofMap.isMatIndexFromGlobal() ?
dofMap.getMatIndex(j, colDofIndex.global) :
dofMap.getMatIndex(j, col(*icursor)));
if (value(*icursor) != 0.0 || petscRowIdx == petscColIdx) {
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the diagNnz values for this row,
// otherwise the offdiagNnz value.
if (petscColIdx >= rankStartIndex &&
petscColIdx < rankStartIndex + nRankRows)
diagNnz[localPetscRowIdx]++;
else
offdiagNnz[localPetscRowIdx]++;
}
int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
colDofMap.getMatIndex(j, colDofIndex.global) :
colDofMap.getMatIndex(j, col(*icursor)));
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the diagNnz values for this row,
// otherwise the offdiagNnz value.
if (petscColIdx >= rankStartColIndex &&
petscColIdx < rankStartColIndex + nRankCols)
nnz.dnnz[localPetscRowIdx]++;
else
nnz.onnz[localPetscRowIdx]++;
}
}
} else {
......@@ -1389,17 +1384,15 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (dofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
if (colDofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
continue;
if (value(*icursor) != 0.0) {
int petscColIdx = (dofMap.isMatIndexFromGlobal() ?
dofMap.getMatIndex(j, colDofIndex.global) :
dofMap.getMatIndex(j, col(*icursor)));
sendMatrixEntry[sendToRank].
push_back(make_pair(petscRowIdx, petscColIdx));
}
int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
colDofMap.getMatIndex(j, colDofIndex.global) :
colDofMap.getMatIndex(j, col(*icursor)));
sendMatrixEntry[sendToRank].
push_back(make_pair(petscRowIdx, petscColIdx));
}
} // if (isRankDof[*cursor]) ... else ...
......@@ -1407,6 +1400,7 @@ namespace AMDiS {
}
}
if (localMatrix == false) {
// === Send and recv the nnz row structure to/from other ranks. ===
......@@ -1428,43 +1422,54 @@ namespace AMDiS {
int r = it->second[i].first;
int c = it->second[i].second;
int localRowIdx = r - rankStartIndex;
int localRowIdx = r - rankStartRowIndex;
TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows)
("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n",
r, localRowIdx, nRankRows, it->first);
if (c < rankStartIndex || c >= rankStartIndex + nRankRows)
offdiagNnz[localRowIdx]++;
if (c < rankStartColIndex || c >= rankStartColIndex + nRankCols)
nnz.onnz[localRowIdx]++;
else
diagNnz[localRowIdx]++;
nnz.dnnz[localRowIdx]++;
}
}
}
// The above algorithm for calculating the number of nnz per row over-
// approximates the value, i.e., the number is always equal or larger to
// the real number of nnz values in the global parallel matrix. For small
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
if (nRankRows < 100)
for (int i = 0; i < nRankRows; i++)
diagNnz[i] = std::min(diagNnz[i], nRankRows);
}
// The above algorithm for calculating the number of nnz per row over-
// approximates the value, i.e., the number is always equal or larger to
// the real number of nnz values in the global parallel matrix. For small
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
// if (nRankRows < 100)
// for (int i = 0; i < nRankRows; i++)
// nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankCols);
#if (DEBUG != 0)
#if (DEBUG != 0)
int nMax = 0;
int nSum = 0;
for (int i = 0; i < nRankRows; i++) {
nMax = std::max(nMax, diagNnz[i]);
nSum += diagNnz[i];
nMax = std::max(nMax, nnz.dnnz[i]);
nSum += nnz.dnnz[i];
}
MSG("NNZ in matrix: max = %d, avrg = %.0f\n",
nMax, static_cast<double>(nSum) / nRankRows);
#endif
MSG("NNZ in diag block: max = %d, avrg = %.0f\n",
nMax, (nSum > 0 ? (static_cast<double>(nSum) / nRankRows) : 0));
if (!localMatrix) {
nMax = 0;
nSum = 0;
for (int i = 0; i < nRankRows; i++) {
nMax = std::max(nMax, nnz.onnz[i]);
nSum += nnz.onnz[i];
}
MSG("NNZ in offdiag block: max = %d, avrg = %.0f\n",
nMax, (nSum > 0 ? (static_cast<double>(nSum) / nRankRows) : 0));
}
}
#endif
}
AMDiS/src/parallel/PetscSolverGlobalMatrix.h
View file @ 2bbf10f7
......@@ -31,16 +31,56 @@ namespace AMDiS {
using namespace std;
struct NnzStructure {
NnzStructure()
: dnnz(NULL),
onnz(NULL)
{}
void clear()
{
if (dnnz) {
delete [] dnnz;
dnnz = NULL;
}
if (onnz) {
delete [] onnz;
onnz = NULL;
}
}
void create(int nRows0, int nRows1 = -1)
{
if (nRows0 == 0)
return;
TEST_EXIT_DBG(nRows0 > 0)("Should not happen!\n");
dnnz = new int[nRows0];
for (int i = 0; i < nRows0; i++)
dnnz[i] = 0;
if (nRows1 > 0) {
onnz = new int[nRows1];
for (int i = 0; i < nRows1; i++)
onnz[i] = 0;
}
}
int *dnnz;
int *onnz;
};
class PetscSolverGlobalMatrix : public PetscSolver
{
public:
PetscSolverGlobalMatrix()
: PetscSolver(),
petscSolVec(PETSC_NULL),
matInteriorDiagNnz(NULL),
matInteriorOffdiagNnz(NULL),
matCoarseDiagNnz(NULL),
matCoarseOffdiagNnz(NULL),
lastMeshNnz(0),
zeroStartVector(false),
alwaysCreateNnzStructure(false)
......@@ -81,9 +121,9 @@ namespace AMDiS {
void createPetscNnzStructure(Matrix<DOFMatrix*> *mat);
void createPetscNnzStructureWithCoarseSpace(Matrix<DOFMatrix*> *mat,
ParallelDofMapping &dofMap,
int*& diagNnz,
int*& offdiagNnz,
ParallelDofMapping &rowDofMap,
ParallelDofMapping &colDofMap,
NnzStructure &nnz,
bool localMatrix = false);
/// Takes a DOF matrix and sends the values to the global PETSc matrix.
......@@ -108,11 +148,11 @@ namespace AMDiS {
protected:
/// Arrays definig the non zero pattern of Petsc's interior matrix.
int *matInteriorDiagNnz, *matInteriorOffdiagNnz;
NnzStructure nnzInterior;
NnzStructure nnzCoarse;
/// Arrays definig the non zero pattern of Petsc's coarse space matrix.
int *matCoarseDiagNnz, *matCoarseOffdiagNnz;
NnzStructure nnzIntCoarse, nnzCoarseInt;
Vec petscSolVec;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment