Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 2bbf10f7 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed some problems with nnz computation with coarse grid, still not finished.

parent 53c5f965
......@@ -51,7 +51,9 @@ namespace AMDiS {
MatCreateAIJ(mpiCommGlobal, nRankRows, nRankRows,
nOverallRows, nOverallRows,
0, matInteriorDiagNnz, 0, matInteriorOffdiagNnz, &matIntInt);
0, nnzInterior.dnnz,
0, nnzInterior.onnz,
&matIntInt);
MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
......@@ -134,26 +136,38 @@ namespace AMDiS {
if (checkMeshChange(mat, localMatrix)) {
createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap,
matInteriorDiagNnz,
matInteriorOffdiagNnz,
*interiorMap,
nnzInterior,
localMatrix);
if (coarseSpaceMap)
if (coarseSpaceMap) {
createPetscNnzStructureWithCoarseSpace(mat,
*coarseSpaceMap,
*coarseSpaceMap,
nnzCoarse);
createPetscNnzStructureWithCoarseSpace(mat,
*coarseSpaceMap,
matCoarseDiagNnz,
matCoarseOffdiagNnz);
*interiorMap,
nnzCoarseInt);
createPetscNnzStructureWithCoarseSpace(mat,
*interiorMap,
*coarseSpaceMap,
nnzIntCoarse);
}
}
if (localMatrix) {
MatCreateSeqAIJ(mpiCommLocal, nRowsRankInterior, nRowsRankInterior,
0, matInteriorDiagNnz, &matIntInt);
0, nnzInterior.dnnz,
&matIntInt);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} else {
MatCreateAIJ(mpiCommLocal,
nRowsRankInterior, nRowsRankInterior,
nRowsOverallInterior, nRowsOverallInterior,
0, matInteriorDiagNnz, 0, matInteriorOffdiagNnz,
0, nnzInterior.dnnz, 0, nnzInterior.onnz,
&matIntInt);
// MatSetOption(matIntInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
if (coarseSpaceMap) {
......@@ -163,13 +177,16 @@ namespace AMDiS {
MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankCoarse,
nRowsOverallCoarse, nRowsOverallCoarse,
0, matCoarseDiagNnz, 0, matCoarseOffdiagNnz,
0, nnzCoarse.dnnz, 0, nnzCoarse.onnz,
// 100, PETSC_NULL, 100, PETSC_NULL,
&matCoarseCoarse);
// MatSetOption(matCoarseCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankInterior,
nRowsOverallCoarse, nGlobalOverallInterior,
100, PETSC_NULL, 100, PETSC_NULL,
// 0, nnzCoarseInt.dnnz, 0, nnzCoarseInt.onnz,
&matCoarseInt);
MatSetOption(matCoarseInt, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
......@@ -177,7 +194,7 @@ namespace AMDiS {
nRowsRankInterior, nRowsRankCoarse,
nGlobalOverallInterior, nRowsOverallCoarse,
100, PETSC_NULL, 100, PETSC_NULL,
// 0, matInteriorDiagNnz, 0, matCoarseOffdiagNnz,
// 0, nnzIntCoarse.dnnz, 0, nnzIntCoarse.onnz,
&matIntCoarse);
MatSetOption(matIntCoarse, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
......@@ -259,8 +276,13 @@ namespace AMDiS {
if (colsOther.size()) {
if (subdomainLevel == 0) {
for (unsigned int k = 0; k < colsOther.size(); k++)
for (unsigned int k = 0; k < colsOther.size(); k++) {
int local = interiorMap->getMatIndex(j, colsOther[k]) - interiorMap->getStartDofs();
if (rowIndex == 0 && local == 9297)
MSG("FOUND: %d %d %d\n", colsOther[k], j, interiorMap->getMatIndex(j, colsOther[k]));
colsOther[k] = interiorMap->getMatIndex(j, colsOther[k]);
}
} else {
for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] =
......@@ -318,6 +340,8 @@ namespace AMDiS {
MatAssemblyEnd(matCoarseInt, MAT_FINAL_ASSEMBLY);
}
AMDiS::finalize();
exit(0);
// === Remove Dirichlet BC DOFs. ===
......@@ -577,30 +601,18 @@ namespace AMDiS {
static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
mpiCommGlobal.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
if (!matInteriorDiagNnz || recvAllValues != 0 || alwaysCreateNnzStructure) {
if (!nnzInterior.dnnz || recvAllValues != 0 || alwaysCreateNnzStructure) {
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
interiorMap->setComputeMatIndex(true, !localMatrix);
interiorMap->update(feSpaces);
if (matInteriorDiagNnz) {
delete [] matInteriorDiagNnz;
matInteriorDiagNnz = NULL;
}
if (matInteriorOffdiagNnz) {
delete [] matInteriorOffdiagNnz;
matInteriorOffdiagNnz = NULL;
}
if (matCoarseDiagNnz) {
delete [] matCoarseDiagNnz;
matCoarseDiagNnz = NULL;
}
nnzInterior.clear();
if (matCoarseOffdiagNnz) {
delete [] matCoarseOffdiagNnz;
matCoarseOffdiagNnz = NULL;
if (coarseSpaceMap) {
nnzCoarse.clear();
nnzCoarseInt.clear();
nnzIntCoarse.clear();
}
updateSubdomainData();
......@@ -1034,18 +1046,11 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()");
TEST_EXIT_DBG(!matInteriorDiagNnz)("There is something wrong!\n");
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
int nRankRows = interiorMap->getRankDofs();
int rankStartIndex = interiorMap->getStartDofs();
matInteriorDiagNnz = new int[nRankRows];
matInteriorOffdiagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++) {
matInteriorDiagNnz[i] = 0;
matInteriorOffdiagNnz[i] = 0;
}
nnzInterior.create(nRankRows, nRankRows);
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits;
......@@ -1143,9 +1148,9 @@ namespace AMDiS {
// otherwise the matInteriorDiagNnz value.
if (petscColIdx >= rankStartIndex &&
petscColIdx < rankStartIndex + nRankRows)
matInteriorDiagNnz[localPetscRowIdx]++;
nnzInterior.dnnz[localPetscRowIdx]++;
else
matInteriorOffdiagNnz[localPetscRowIdx]++;
nnzInterior.onnz[localPetscRowIdx]++;
}
}
} else {
......@@ -1202,9 +1207,9 @@ namespace AMDiS {
r, localRowIdx, nRankRows, it->first);
if (c < rankStartIndex || c >= rankStartIndex + nRankRows)
matInteriorOffdiagNnz[localRowIdx]++;
nnzInterior.onnz[localRowIdx]++;
else
matInteriorDiagNnz[localRowIdx]++;
nnzInterior.dnnz[localRowIdx]++;
}
}
}
......@@ -1217,33 +1222,26 @@ namespace AMDiS {
if (nRankRows < 100)
for (int i = 0; i < nRankRows; i++)
matInteriorDiagNnz[i] = std::min(matInteriorDiagNnz[i], nRankRows);
nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankRows);
}
void PetscSolverGlobalMatrix::createPetscNnzStructureWithCoarseSpace(Matrix<DOFMatrix*> *mat,
ParallelDofMapping &dofMap,
int*& diagNnz,
int*& offdiagNnz,
ParallelDofMapping &rowDofMap,
ParallelDofMapping &colDofMap,
NnzStructure &nnz,
bool localMatrix)
{
FUNCNAME("PetscSolverGlobalMatrix::createPetscNnzStructure()");
TEST_EXIT_DBG(!diagNnz)("There is something wrong!\n");
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
int nRankRows = dofMap.getRankDofs();
int rankStartIndex = dofMap.getStartDofs();
int nRankRows = rowDofMap.getRankDofs();
int rankStartRowIndex = rowDofMap.getStartDofs();
diagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++)
diagNnz[i] = 0;
int nRankCols = colDofMap.getRankDofs();
int rankStartColIndex = colDofMap.getStartDofs();
if (localMatrix == false) {
offdiagNnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++)
offdiagNnz[i] = 0;
}
nnz.create(nRankRows, (!localMatrix ? nRankRows : -1));
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits;
......@@ -1308,21 +1306,21 @@ namespace AMDiS {
for (cursor_type cursor = begin<row>(bmat),
cend = end<row>(bmat); cursor != cend; ++cursor) {
if (dofMap[feSpaces[i]].find(*cursor, rowDofIndex) == false)
if (rowDofMap[feSpaces[i]].find(*cursor, rowDofIndex) == false)
continue;
// The corresponding global matrix row index of the current row DOF.
int petscRowIdx = 0;
if (localMatrix) {
petscRowIdx = dofMap.getLocalMatIndex(i, *cursor);
petscRowIdx = rowDofMap.getLocalMatIndex(i, *cursor);
} else {
if (dofMap.isMatIndexFromGlobal())
petscRowIdx = dofMap.getMatIndex(i, rowDofIndex.global);
if (rowDofMap.isMatIndexFromGlobal())
petscRowIdx = rowDofMap.getMatIndex(i, rowDofIndex.global);
else
petscRowIdx = dofMap.getMatIndex(i, *cursor);
petscRowIdx = rowDofMap.getMatIndex(i, *cursor);
}
if (localMatrix || dofMap[feSpaces[i]].isRankDof(*cursor)) {
if (localMatrix || rowDofMap[feSpaces[i]].isRankDof(*cursor)) {
// === The current row DOF is a rank DOF, so create the ===
// === corresponding nnz values directly on rank's nnz data. ===
......@@ -1330,50 +1328,47 @@ namespace AMDiS {
// This is the local row index of the local PETSc matrix.
int localPetscRowIdx = petscRowIdx;
if (localMatrix == false) {
localPetscRowIdx -= rankStartIndex;
if (localMatrix == false)
localPetscRowIdx -= rankStartRowIndex;
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows)
("Should not happen! \n Debug info: DOF = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d compontens = %d from %d nRankRows = %d\n",
*cursor,
dofMap[feSpaces[i]][*cursor].global,
rowDofMap[feSpaces[i]][*cursor].global,
petscRowIdx,
localPetscRowIdx,
rankStartIndex,
rankStartRowIndex,
i,
nComponents,
nRankRows);
}
if (localMatrix) {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor)
if (dofMap[feSpaces[j]].isSet(col(*icursor)))
diagNnz[localPetscRowIdx]++;
if (colDofMap[feSpaces[j]].isSet(col(*icursor)))
nnz.dnnz[localPetscRowIdx]++;
} else {
MultiIndex colDofIndex;
// Traverse all non zero entries in this row.
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (dofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
if (colDofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
continue;
int petscColIdx = (dofMap.isMatIndexFromGlobal() ?
dofMap.getMatIndex(j, colDofIndex.global) :
dofMap.getMatIndex(j, col(*icursor)));
int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
colDofMap.getMatIndex(j, colDofIndex.global) :
colDofMap.getMatIndex(j, col(*icursor)));
if (value(*icursor) != 0.0 || petscRowIdx == petscColIdx) {
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the diagNnz values for this row,
// otherwise the offdiagNnz value.
if (petscColIdx >= rankStartIndex &&
petscColIdx < rankStartIndex + nRankRows)
diagNnz[localPetscRowIdx]++;
if (petscColIdx >= rankStartColIndex &&
petscColIdx < rankStartColIndex + nRankCols)
nnz.dnnz[localPetscRowIdx]++;
else
offdiagNnz[localPetscRowIdx]++;
}
nnz.onnz[localPetscRowIdx]++;
}
}
} else {
......@@ -1389,24 +1384,23 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (dofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
if (colDofMap[feSpaces[j]].find(col(*icursor), colDofIndex) == false)
continue;
if (value(*icursor) != 0.0) {
int petscColIdx = (dofMap.isMatIndexFromGlobal() ?
dofMap.getMatIndex(j, colDofIndex.global) :
dofMap.getMatIndex(j, col(*icursor)));
int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
colDofMap.getMatIndex(j, colDofIndex.global) :
colDofMap.getMatIndex(j, col(*icursor)));
sendMatrixEntry[sendToRank].
push_back(make_pair(petscRowIdx, petscColIdx));
}
}
} // if (isRankDof[*cursor]) ... else ...
} // for each row in mat[i][j]
}
}
if (localMatrix == false) {
// === Send and recv the nnz row structure to/from other ranks. ===
......@@ -1428,16 +1422,17 @@ namespace AMDiS {
int r = it->second[i].first;
int c = it->second[i].second;
int localRowIdx = r - rankStartIndex;
int localRowIdx = r - rankStartRowIndex;
TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows)
("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n",
r, localRowIdx, nRankRows, it->first);
if (c < rankStartIndex || c >= rankStartIndex + nRankRows)
offdiagNnz[localRowIdx]++;
if (c < rankStartColIndex || c >= rankStartColIndex + nRankCols)
nnz.onnz[localRowIdx]++;
else
diagNnz[localRowIdx]++;
nnz.dnnz[localRowIdx]++;
}
}
}
}
......@@ -1448,23 +1443,33 @@ namespace AMDiS {
// matrices, the problem may arise, that the result is larger than the
// number of elements in a row. This is fixed in the following.
if (nRankRows < 100)
for (int i = 0; i < nRankRows; i++)
diagNnz[i] = std::min(diagNnz[i], nRankRows);
}
// if (nRankRows < 100)
// for (int i = 0; i < nRankRows; i++)
// nnzInterior.dnnz[i] = std::min(nnzInterior.dnnz[i], nRankCols);
#if (DEBUG != 0)
int nMax = 0;
int nSum = 0;
for (int i = 0; i < nRankRows; i++) {
nMax = std::max(nMax, diagNnz[i]);
nSum += diagNnz[i];
nMax = std::max(nMax, nnz.dnnz[i]);
nSum += nnz.dnnz[i];
}
MSG("NNZ in matrix: max = %d, avrg = %.0f\n",
nMax, static_cast<double>(nSum) / nRankRows);
#endif
MSG("NNZ in diag block: max = %d, avrg = %.0f\n",
nMax, (nSum > 0 ? (static_cast<double>(nSum) / nRankRows) : 0));
if (!localMatrix) {
nMax = 0;
nSum = 0;
for (int i = 0; i < nRankRows; i++) {
nMax = std::max(nMax, nnz.onnz[i]);
nSum += nnz.onnz[i];
}
MSG("NNZ in offdiag block: max = %d, avrg = %.0f\n",
nMax, (nSum > 0 ? (static_cast<double>(nSum) / nRankRows) : 0));
}
}
#endif
}
......@@ -31,16 +31,56 @@ namespace AMDiS {
using namespace std;
struct NnzStructure {
NnzStructure()
: dnnz(NULL),
onnz(NULL)
{}
void clear()
{
if (dnnz) {
delete [] dnnz;
dnnz = NULL;
}
if (onnz) {
delete [] onnz;
onnz = NULL;
}
}
void create(int nRows0, int nRows1 = -1)
{
if (nRows0 == 0)
return;
TEST_EXIT_DBG(nRows0 > 0)("Should not happen!\n");
dnnz = new int[nRows0];
for (int i = 0; i < nRows0; i++)
dnnz[i] = 0;
if (nRows1 > 0) {
onnz = new int[nRows1];
for (int i = 0; i < nRows1; i++)
onnz[i] = 0;
}
}
int *dnnz;
int *onnz;
};
class PetscSolverGlobalMatrix : public PetscSolver
{
public:
PetscSolverGlobalMatrix()
: PetscSolver(),
petscSolVec(PETSC_NULL),
matInteriorDiagNnz(NULL),
matInteriorOffdiagNnz(NULL),
matCoarseDiagNnz(NULL),
matCoarseOffdiagNnz(NULL),
lastMeshNnz(0),
zeroStartVector(false),
alwaysCreateNnzStructure(false)
......@@ -81,9 +121,9 @@ namespace AMDiS {
void createPetscNnzStructure(Matrix<DOFMatrix*> *mat);
void createPetscNnzStructureWithCoarseSpace(Matrix<DOFMatrix*> *mat,
ParallelDofMapping &dofMap,
int*& diagNnz,
int*& offdiagNnz,
ParallelDofMapping &rowDofMap,
ParallelDofMapping &colDofMap,
NnzStructure &nnz,
bool localMatrix = false);
/// Takes a DOF matrix and sends the values to the global PETSc matrix.
......@@ -108,11 +148,11 @@ namespace AMDiS {
protected:
/// Arrays definig the non zero pattern of Petsc's interior matrix.
int *matInteriorDiagNnz, *matInteriorOffdiagNnz;
NnzStructure nnzInterior;
NnzStructure nnzCoarse;
/// Arrays definig the non zero pattern of Petsc's coarse space matrix.
int *matCoarseDiagNnz, *matCoarseOffdiagNnz;
NnzStructure nnzIntCoarse, nnzCoarseInt;
Vec petscSolVec;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment