Commit 2bbf10f7 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed some problems with nnz computation with coarse grid, still not finished.

parent 53c5f965
...@@ -51,7 +51,9 @@ namespace AMDiS { ...@@ -51,7 +51,9 @@ namespace AMDiS {
MatCreateAIJ(mpiCommGlobal, nRankRows, nRankRows, MatCreateAIJ(mpiCommGlobal, nRankRows, nRankRows,
nOverallRows, nOverallRows, nOverallRows, nOverallRows,
0, matInteriorDiagNnz, 0, matInteriorOffdiagNnz, &matIntInt); 0, nnzInterior.dnnz,
0, nnzInterior.onnz,
&matIntInt);
MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
...@@ -132,28 +134,40 @@ namespace AMDiS { ...@@ -132,28 +134,40 @@ namespace AMDiS {
bool localMatrix = (subdomainLevel == 0); bool localMatrix = (subdomainLevel == 0);
if (checkMeshChange(mat, localMatrix)) { if (checkMeshChange(mat, localMatrix)) {
createPetscNnzStructureWithCoarseSpace(mat, createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap, *interiorMap,
matInteriorDiagNnz, *interiorMap,
matInteriorOffdiagNnz, nnzInterior,
localMatrix); localMatrix);
if (coarseSpaceMap) if (coarseSpaceMap) {
createPetscNnzStructureWithCoarseSpace(mat,
*coarseSpaceMap,
*coarseSpaceMap,
nnzCoarse);
createPetscNnzStructureWithCoarseSpace(mat, createPetscNnzStructureWithCoarseSpace(mat,
*coarseSpaceMap, *coarseSpaceMap,
matCoarseDiagNnz, *interiorMap,
matCoarseOffdiagNnz); nnzCoarseInt);
createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap,
*coarseSpaceMap,
nnzIntCoarse);
}
} }
if (localMatrix) { if (localMatrix) {
MatCreateSeqAIJ(mpiCommLocal, nRowsRankInterior, nRowsRankInterior, MatCreateSeqAIJ(mpiCommLocal, nRowsRankInterior, nRowsRankInterior,
0, matInteriorDiagNnz, &matIntInt); 0, nnzInterior.dnnz,
&matIntInt);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} else { } else {
MatCreateAIJ(mpiCommLocal, MatCreateAIJ(mpiCommLocal,
nRowsRankInterior, nRowsRankInterior, nRowsRankInterior, nRowsRankInterior,
nRowsOverallInterior, nRowsOverallInterior, nRowsOverallInterior, nRowsOverallInterior,
0, matInteriorDiagNnz, 0, matInteriorOffdiagNnz, 0, nnzInterior.dnnz, 0, nnzInterior.onnz,
&matIntInt); &matIntInt);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} }
if (coarseSpaceMap) { if (coarseSpaceMap) {
...@@ -163,13 +177,16 @@ namespace AMDiS { ...@@ -163,13 +177,16 @@ namespace AMDiS {
MatCreateAIJ(mpiCommGlobal, MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankCoarse, nRowsRankCoarse, nRowsRankCoarse,
nRowsOverallCoarse, nRowsOverallCoarse, nRowsOverallCoarse, nRowsOverallCoarse,
0, matCoarseDiagNnz, 0, matCoarseOffdiagNnz, 0, nnzCoarse.dnnz, 0, nnzCoarse.onnz,
// 100, PETSC_NULL, 100, PETSC_NULL,
&matCoarseCoarse); &matCoarseCoarse);
// MatSetOption(matCoarseCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
MatCreateAIJ(mpiCommGlobal, MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankInterior, nRowsRankCoarse, nRowsRankInterior,
nRowsOverallCoarse, nGlobalOverallInterior, nRowsOverallCoarse, nGlobalOverallInterior,
100, PETSC_NULL, 100, PETSC_NULL, 100, PETSC_NULL, 100, PETSC_NULL,
// 0, nnzCoarseInt.dnnz, 0, nnzCoarseInt.onnz,
&matCoarseInt); &matCoarseInt);
MatSetOption(matCoarseInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); MatSetOption(matCoarseInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
...@@ -177,7 +194,7 @@ namespace AMDiS { ...@@ -177,7 +194,7 @@ namespace AMDiS {
nRowsRankInterior, nRowsRankCoarse, nRowsRankInterior, nRowsRankCoarse,
nGlobalOverallInterior, nRowsOverallCoarse, nGlobalOverallInterior, nRowsOverallCoarse,
100, PETSC_NULL, 100, PETSC_NULL, 100, PETSC_NULL, 100, PETSC_NULL,
// 0, matInteriorDiagNnz, 0, matCoarseOffdiagNnz, // 0, nnzIntCoarse.dnnz, 0, nnzIntCoarse.onnz,
&matIntCoarse); &matIntCoarse);
MatSetOption(matIntCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); MatSetOption(matIntCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} }
...@@ -259,14 +276,19 @@ namespace AMDiS { ...@@ -259,14 +276,19 @@ namespace AMDiS {
if (colsOther.size()) { if (colsOther.size()) {
if (subdomainLevel == 0) { if (subdomainLevel == 0) {
for (unsigned int k = 0; k < colsOther.size(); k++) for (unsigned int k = 0; k < colsOther.size(); k++) {
int local = interiorMap->getMatIndex(j, colsOther[k]) - interiorMap->getStartDofs();
if (rowIndex == 0 && local == 9297)
MSG("FOUND: %d %d %d\n", colsOther[k], j, interiorMap->getMatIndex(j, colsOther[k]));
colsOther[k] = interiorMap->getMatIndex(j, colsOther[k]); colsOther[k] = interiorMap->getMatIndex(j, colsOther[k]);
}
} else { } else {
for (unsigned int k = 0; k < colsOther.size(); k++) for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] = colsOther[k] =
interiorMap->getMatIndex(j, colsOther[k]) + rStartInterior; interiorMap->getMatIndex(j, colsOther[k]) + rStartInterior;
} }
MatSetValues(matCoarseInt, 1, &rowIndex, colsOther.size(), MatSetValues(matCoarseInt, 1, &rowIndex, colsOther.size(),
&(colsOther[0]), &(valuesOther[0]), ADD_VALUES); &(colsOther[0]), &(valuesOther[0]), ADD_VALUES);
} }
...@@ -318,6 +340,8 @@ namespace AMDiS { ...@@ -318,6 +340,8 @@ namespace AMDiS {
MatAssemblyEnd(matCoarseInt, MAT_FINAL_ASSEMBLY); MatAssemblyEnd(matCoarseInt, MAT_FINAL_ASSEMBLY);
} }
AMDiS::finalize();
exit(0);
// === Remove Dirichlet BC DOFs. === // === Remove Dirichlet BC DOFs. ===
...@@ -577,30 +601,18 @@ namespace AMDiS { ...@@ -577,30 +601,18 @@ namespace AMDiS {
static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz); static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
mpiCommGlobal.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM); mpiCommGlobal.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
if (!matInteriorDiagNnz || recvAllValues != 0 || alwaysCreateNnzStructure) { if (!nnzInterior.dnnz || recvAllValues != 0 || alwaysCreateNnzStructure) {
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat); vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
interiorMap->setComputeMatIndex(true, !localMatrix); interiorMap->setComputeMatIndex(true, !localMatrix);
interiorMap->update(feSpaces); interiorMap->update(feSpaces);
if (matInteriorDiagNnz) { nnzInterior.clear();
delete [] matInteriorDiagNnz;
matInteriorDiagNnz = NULL;
}
if (matInteriorOffdiagNnz) {
delete [] matInteriorOffdiagNnz;
matInteriorOffdiagNnz = NULL;
}
if (matCoarseDiagNnz) { if (coarseSpaceMap) {
delete [] matCoarseDiagNnz; nnzCoarse.clear();
matCoarseDiagNnz = NULL; nnzCoarseInt.clear();
} nnzIntCoarse.clear();
if (matCoarseOffdiagNnz) {
delete [] matCoarseOffdiagNnz;
matCoarseOffdiagNnz = NULL;
} }
updateSubdomainData(); updateSubdomainData();
...@@ -1034,18 +1046,11 @@ namespace AMDiS { ...@@ -1034,18 +1046,11 @@ namespace AMDiS {
{ {
FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()"); FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()");
TEST_EXIT_DBG(!matInteriorDiagNnz)("There is something wrong!\n");
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat); vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
int nRankRows = interiorMap->getRankDofs(); int nRankRows = interiorMap->getRankDofs();
int rankStartIndex = interiorMap->getStartDofs(); int rankStartIndex = interiorMap->getStartDofs();
matInteriorDiagNnz = new int[nRankRows]; nnzInterior.create(nRankRows, nRankRows);
matInteriorOffdiagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++) {
matInteriorDiagNnz[i] = 0;
matInteriorOffdiagNnz[i] = 0;
}
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end; using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits; namespace traits = mtl::traits;
...@@ -1143,9 +1148,9 @@ namespace AMDiS { ...@@ -1143,9 +1148,9 @@ namespace AMDiS {
// otherwise the matInteriorDiagNnz value. // otherwise the matInteriorDiagNnz value.
if (petscColIdx >= rankStartIndex && if (petscColIdx >= rankStartIndex &&
petscColIdx < rankStartIndex + nRankRows) petscColIdx < rankStartIndex + nRankRows)
matInteriorDiagNnz[localPetscRowIdx]++; nnzInterior.dnnz[localPetscRowIdx]++;
else else
matInteriorOffdiagNnz[localPetscRowIdx]++; nnzInterior.onnz[localPetscRowIdx]++;
} }
} }
} else { } else {
...@@ -1202,9 +1207,9 @@ namespace AMDiS { ...@@ -1202,9 +1207,9 @@ namespace AMDiS {
r, localRowIdx, nRankRows, it->first); r, localRowIdx, nRankRows, it->first);
if (c < rankStartIndex || c >= rankStartIndex + nRankRows) if (c < rankStartIndex || c >= rankStartIndex + nRankRows)
matInteriorOffdiagNnz[localRowIdx]++; nnzInterior.onnz[localRowIdx]++;
else else
matInteriorDiagNnz[localRowIdx]++; nnzInterior.dnnz[localRowIdx]++;
} }
} }
} }
...@@ -1217,33 +1222,26 @@ namespace AMDiS { ...@@ -1217,33 +1222,26 @@ namespace AMDiS {
if (nRankRows < 100) if (nRankRows < 100)
for (int i = 0; i < nRankRows; i++) for (int i = 0; i < nRankRows; i++)
matInteriorDiagNnz[i] = std::min(matInteriorDiagNnz[i], nRankRows); nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankRows);
} }
void PetscSolverGlobalMatrix::createPetscNnzStructureWithCoarseSpace(Matrix<DOFMatrix*> *mat, void PetscSolverGlobalMatrix::createPetscNnzStructureWithCoarseSpace(Matrix<DOFMatrix*> *mat,
ParallelDofMapping &dofMap, ParallelDofMapping &rowDofMap,
int*& diagNnz, ParallelDofMapping &colDofMap,
int*& offdiagNnz, NnzStructure &nnz,
bool localMatrix) bool localMatrix)
{ {
FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()"); FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()");
TEST_EXIT_DBG(!diagNnz)("There is something wrong!\n");
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat); vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
int nRankRows = dofMap.getRankDofs(); int nRankRows = rowDofMap.getRankDofs();
int rankStartIndex = dofMap.getStartDofs(); int rankStartRowIndex = rowDofMap.getStartDofs();
diagNnz = new int[nRankRows]; int nRankCols = colDofMap.getRankDofs();
for (int i = 0; i < nRankRows; i++) int rankStartColIndex = colDofMap.getStartDofs();
diagNnz[i] = 0;
if (localMatrix == false) { nnz.create(nRankRows, (!localMatrix ? nRankRows : -1));
offdiagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++)
offdiagNnz[i] = 0;
}
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end; using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits; namespace traits = mtl::traits;
...@@ -1308,21 +1306,21 @@ namespace AMDiS { ...@@ -1308,21 +1306,21 @@ namespace AMDiS {
for (cursor_type cursor = begin<row>(bmat), for (cursor_type cursor = begin<row>(bmat),
cend = end<row>(bmat); cursor != cend; ++cursor) { cend = end<row>(bmat); cursor != cend; ++cursor) {
if (dofMap[feSpaces[i]].find(*cursor, rowDofIndex) == false) if (rowDofMap[feSpaces[i]].find(*cursor, rowDofIndex) == false)
continue; continue;
// The corresponding global matrix row index of the current row DOF. // The corresponding global matrix row index of the current row DOF.
int petscRowIdx = 0; int petscRowIdx = 0;
if (localMatrix) { if (localMatrix) {
petscRowIdx = dofMap.getLocalMatIndex(i, *cursor); petscRowIdx = rowDofMap.getLocalMatIndex(i, *cursor);
} else { } else {
if (dofMap.isMatIndexFromGlobal()) if (rowDofMap.isMatIndexFromGlobal())
petscRowIdx = dofMap.getMatIndex(i, rowDofIndex.global); petscRowIdx = rowDofMap.getMatIndex(i, rowDofIndex.global);
else else
petscRowIdx = dofMap.getMatIndex(i, *cursor); petscRowIdx = rowDofMap.getMatIndex(i, *cursor);
} }
if (localMatrix || dofMap[feSpaces[i]].isRankDof(*cursor)) { if (localMatrix || rowDofMap[feSpaces[i]].isRankDof(*cursor)) {
// === The current row DOF is a rank DOF, so create the === // === The current row DOF is a rank DOF, so create the ===
// === corresponding nnz values directly on rank's nnz data. === // === corresponding nnz values directly on rank's nnz data. ===
...@@ -1330,50 +1328,47 @@ namespace AMDiS { ...@@ -1330,50 +1328,47 @@ namespace AMDiS {
// This is the local row index of the local PETSc matrix. // This is the local row index of the local PETSc matrix.
int localPetscRowIdx = petscRowIdx; int localPetscRowIdx = petscRowIdx;
if (localMatrix == false) { if (localMatrix == false)
localPetscRowIdx -= rankStartIndex; localPetscRowIdx -= rankStartRowIndex;
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows) TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows)
("Should not happen! \n Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d\n", ("Should not happen! \n Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d\n",
*cursor, *cursor,
dofMap[feSpaces[i]][*cursor].global, rowDofMap[feSpaces[i]][*cursor].global,
petscRowIdx, petscRowIdx,
localPetscRowIdx, localPetscRowIdx,
rankStartIndex, rankStartRowIndex,
i, i,
nComponents, nComponents,
nRankRows); nRankRows);
}
if (localMatrix) { if (localMatrix) {
for (icursor_type icursor = begin<nz>(cursor), for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) icend = end<nz>(cursor); icursor != icend; ++icursor)
if (dofMap[feSpaces[j]].isSet(col(*icursor))) if (colDofMap[feSpaces[j]].isSet(col(*icursor)))
diagNnz[localPetscRowIdx]++; nnz.dnnz[localPetscRowIdx]++;
} else { } else {
MultiIndex colDofIndex; MultiIndex colDofIndex;
// Traverse all non zero entries in this row. // Traverse all non zero entries in this row.
for (icursor_type icursor = begin<nz>(cursor), for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) { icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (dofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false) if (colDofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
continue; continue;
int petscColIdx = (dofMap.isMatIndexFromGlobal() ? int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
dofMap.getMatIndex(j, colDofIndex.global) : colDofMap.getMatIndex(j, colDofIndex.global) :
dofMap.getMatIndex(j, col(*icursor))); colDofMap.getMatIndex(j, col(*icursor)));
if (value(*icursor) != 0.0 || petscRowIdx == petscColIdx) { // The row DOF is a rank DOF, if also the column is a rank DOF,
// The row DOF is a rank DOF, if also the column is a rank DOF, // increment the diagNnz values for this row,
// increment the diagNnz values for this row, // otherwise the offdiagNnz value.
// otherwise the offdiagNnz value. if (petscColIdx >= rankStartColIndex &&
if (petscColIdx >= rankStartIndex && petscColIdx < rankStartColIndex + nRankCols)
petscColIdx < rankStartIndex + nRankRows) nnz.dnnz[localPetscRowIdx]++;
diagNnz[localPetscRowIdx]++; else
else nnz.onnz[localPetscRowIdx]++;
offdiagNnz[localPetscRowIdx]++;
}
} }
} }
} else { } else {
...@@ -1389,17 +1384,15 @@ namespace AMDiS { ...@@ -1389,17 +1384,15 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor), for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) { icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (dofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false) if (colDofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
continue; continue;
if (value(*icursor) != 0.0) { int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
int petscColIdx = (dofMap.isMatIndexFromGlobal() ? colDofMap.getMatIndex(j, colDofIndex.global) :
dofMap.getMatIndex(j, colDofIndex.global) : colDofMap.getMatIndex(j, col(*icursor)));
dofMap.getMatIndex(j, col(*icursor)));
sendMatrixEntry[sendToRank].
sendMatrixEntry[sendToRank]. push_back(make_pair(petscRowIdx, petscColIdx));
push_back(make_pair(petscRowIdx, petscColIdx));
}
} }
} // if (isRankDof[*cursor]) ... else ... } // if (isRankDof[*cursor]) ... else ...
...@@ -1407,6 +1400,7 @@ namespace AMDiS { ...@@ -1407,6 +1400,7 @@ namespace AMDiS {
} }
} }
if (localMatrix == false) { if (localMatrix == false) {
// === Send and recv the nnz row structure to/from other ranks. === // === Send and recv the nnz row structure to/from other ranks. ===
...@@ -1428,43 +1422,54 @@ namespace AMDiS { ...@@ -1428,43 +1422,54 @@ namespace AMDiS {
int r = it->second[i].first; int r = it->second[i].first;
int c = it->second[i].second; int c = it->second[i].second;
int localRowIdx = r - rankStartIndex; int localRowIdx = r - rankStartRowIndex;
TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows) TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows)
("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n", ("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n",
r, localRowIdx, nRankRows, it->first); r, localRowIdx, nRankRows, it->first);
if (c < rankStartIndex || c >= rankStartIndex + nRankRows) if (c < rankStartColIndex || c >= rankStartColIndex + nRankCols)
offdiagNnz[localRowIdx]++; nnz.onnz[localRowIdx]++;
else else
diagNnz[localRowIdx]++; nnz.dnnz[localRowIdx]++;
} }
} }
} }
// The above algorithm for calculating the number of nnz per row over-
// approximates the value, i.e., the number is always equal or larger to
// the real number of nnz values in the global parallel matrix. For small
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
if (nRankRows < 100)
for (int i = 0; i < nRankRows; i++)
diagNnz[i] = std::min(diagNnz[i], nRankRows);
} }
// The above algorithm for calculating the number of nnz per row over-
// approximates the value, i.e., the number is always equal or larger to
// the real number of nnz values in the global parallel matrix. For small
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
// if (nRankRows < 100)
// for (int i = 0; i < nRankRows; i++)
// nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankCols);
#if (DEBUG != 0) #if (DEBUG != 0)
int nMax = 0; int nMax = 0;
int nSum = 0; int nSum = 0;
for (int i = 0; i < nRankRows; i++) { for (int i = 0; i < nRankRows; i++) {
nMax = std::max(nMax, diagNnz[i]); nMax = std::max(nMax, nnz.dnnz[i]);
nSum += diagNnz[i]; nSum += nnz.dnnz[i];
} }
MSG("NNZ in matrix: max = %d, avrg = %.0f\n", MSG("NNZ in diag block: max = %d, avrg = %.0f\n",
nMax, static_cast<double>(nSum) / nRankRows); nMax, (nSum > 0 ? (static_cast<double>(nSum) / nRankRows) : 0));
#endif
if (!localMatrix) {
nMax = 0;
nSum = 0;
for (int i = 0; i < nRankRows; i++) {
nMax = std::max(nMax, nnz.onnz[i]);
nSum += nnz.onnz[i];
}
MSG("NNZ in offdiag block: max = %d, avrg = %.0f\n",
nMax, (nSum > 0 ? (static_cast<double>(nSum) / nRankRows) : 0));
}
} }
#endif
} }
...@@ -31,16 +31,56 @@ namespace AMDiS { ...@@ -31,16 +31,56 @@ namespace AMDiS {
using namespace std; using namespace std;