Commit d81f254a authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed nnz caluculations for petsc solver.

parent 77dbdc69
...@@ -228,7 +228,14 @@ namespace AMDiS { ...@@ -228,7 +228,14 @@ namespace AMDiS {
bound.rankObj.elIndex)("Should not happen!\n"); bound.rankObj.elIndex)("Should not happen!\n");
bound.rankObj.el = elIndexMap[bound.rankObj.elIndex]; bound.rankObj.el = elIndexMap[bound.rankObj.elIndex];
bound.neighObj.el = NULL;
// For the case of periodic interior boundaries, a rank may have an boundary
// with itself. In this case, also the pointer to the neighbour object must
// be set correctly.
if (elIndexMap.count(bound.neighObj.elIndex))
bound.neighObj.el = elIndexMap[bound.neighObj.elIndex];
else
bound.neighObj.el = NULL;
} }
} }
} }
......
...@@ -1644,6 +1644,10 @@ namespace AMDiS { ...@@ -1644,6 +1644,10 @@ namespace AMDiS {
for (unsigned int i = 0; i < it->second.size(); i++) { for (unsigned int i = 0; i < it->second.size(); i++) {
AtomicBoundary &bound = it->second[i]; AtomicBoundary &bound = it->second[i];
TEST_EXIT_DBG(bound.rankObj.el)("No rank object!\n");
TEST_EXIT_DBG(bound.neighObj.el)("No neigh object!\n");
DofContainer dofs0, dofs1; DofContainer dofs0, dofs1;
bound.rankObj.el->getVertexDofs(feSpace, bound.rankObj, dofs0); bound.rankObj.el->getVertexDofs(feSpace, bound.rankObj, dofs0);
bound.rankObj.el->getNonVertexDofs(feSpace, bound.rankObj, dofs0); bound.rankObj.el->getNonVertexDofs(feSpace, bound.rankObj, dofs0);
......
...@@ -201,7 +201,10 @@ namespace AMDiS { ...@@ -201,7 +201,10 @@ namespace AMDiS {
/// rank's partition, but is owned by some other rank. /// rank's partition, but is owned by some other rank.
inline bool getIsRankDof(DegreeOfFreedom dof) inline bool getIsRankDof(DegreeOfFreedom dof)
{ {
return isRankDof[dof]; if (isRankDof.count(dof))
return isRankDof[dof];
return false;
} }
inline long getLastMeshChangeIndex() inline long getLastMeshChangeIndex()
...@@ -526,7 +529,7 @@ namespace AMDiS { ...@@ -526,7 +529,7 @@ namespace AMDiS {
* If periodic boundaries are used, this map stores to each periodic DOF in rank's * If periodic boundaries are used, this map stores to each periodic DOF in rank's
* partition the set of periodic boundaries the DOF is associated to. In 2D, most * partition the set of periodic boundaries the DOF is associated to. In 2D, most
* DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries * DOFs are only on one periodic boundary. Only, e.g., in a box with all boundaries
* being periodic, the for corners are associated by two different boundaries. * being periodic, the four corners are associated by two different boundaries.
*/ */
map<int, std::set<BoundaryType> > periodicDofAssociations; map<int, std::set<BoundaryType> > periodicDofAssociations;
......
...@@ -143,32 +143,6 @@ namespace AMDiS { ...@@ -143,32 +143,6 @@ namespace AMDiS {
{ {
FUNCNAME("ParallelDebug::testPeriodicBoundary()"); FUNCNAME("ParallelDebug::testPeriodicBoundary()");
DegreeOfFreedom testDof = pdb.mapGlobalToLocal(968);
if (testDof != -1) {
WorldVector<double> c;
pdb.mesh->getDofIndexCoords(testDof, pdb.feSpace, c);
MSG("FOUND: LOCAL %d GLOBAL %d COORDS %f %f %f\n",
testDof, 968, c[0], c[1], c[2]);
}
testDof = pdb.mapGlobalToLocal(973);
if (testDof != -1) {
WorldVector<double> c;
pdb.mesh->getDofIndexCoords(testDof, pdb.feSpace, c);
MSG("FOUND: LOCAL %d GLOBAL %d COORDS %f %f %f\n",
testDof, 973, c[0], c[1], c[2]);
}
testDof = pdb.mapGlobalToLocal(1795);
if (testDof != -1) {
WorldVector<double> c;
pdb.mesh->getDofIndexCoords(testDof, pdb.feSpace, c);
MSG("FOUND: LOCAL %d GLOBAL %d COORDS %f %f %f\n",
testDof, 1795, c[0], c[1], c[2]);
}
// === 1. check: All periodic DOFs should have at least a correct number === // === 1. check: All periodic DOFs should have at least a correct number ===
// === of periodic associations. === // === of periodic associations. ===
......
...@@ -100,6 +100,9 @@ namespace AMDiS { ...@@ -100,6 +100,9 @@ namespace AMDiS {
// Test if the current col dof is a periodic dof. // Test if the current col dof is a periodic dof.
bool periodicCol = meshDistributor->isPeriodicDof(globalColDof); bool periodicCol = meshDistributor->isPeriodicDof(globalColDof);
if (value(*icursor) == 0.0 && globalRowDof != globalColDof)
continue;
if (!periodicCol) { if (!periodicCol) {
// Calculate the exact position of the column index in the petsc matrix. // Calculate the exact position of the column index in the petsc matrix.
cols.push_back(globalColDof * dispMult + dispAddCol); cols.push_back(globalColDof * dispMult + dispAddCol);
...@@ -130,7 +133,7 @@ namespace AMDiS { ...@@ -130,7 +133,7 @@ namespace AMDiS {
} }
} }
for (int i = 0; i < newCols.size(); i++) { for (unsigned int i = 0; i < newCols.size(); i++) {
cols.push_back(newCols[i] * dispMult + dispAddCol); cols.push_back(newCols[i] * dispMult + dispAddCol);
values.push_back(scaledValue); values.push_back(scaledValue);
} }
...@@ -140,12 +143,18 @@ namespace AMDiS { ...@@ -140,12 +143,18 @@ namespace AMDiS {
MatSetValues(petscMatrix, 1, &rowIndex, cols.size(), MatSetValues(petscMatrix, 1, &rowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES); &(cols[0]), &(values[0]), ADD_VALUES);
} else { } else {
std::map<int, std::vector<int> > colsMap;
std::map<int, std::vector<double> > valsMap;
for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor);
icursor != icend; ++icursor) { icursor != icend; ++icursor) {
// Global index of the current column index. // Global index of the current column index.
int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor)); int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor));
if (value(*icursor) == 0.0 && globalRowDof != globalColDof)
continue;
std::set<int> perAsc; std::set<int> perAsc;
if (meshDistributor->isPeriodicDof(globalColDof)) { if (meshDistributor->isPeriodicDof(globalColDof)) {
...@@ -157,14 +166,12 @@ namespace AMDiS { ...@@ -157,14 +166,12 @@ namespace AMDiS {
perAsc.insert(*it); perAsc.insert(*it);
} }
if (meshDistributor->isPeriodicDof(globalRowDof)) { std::set<int>& perRowAsc =
std::set<int>& perRowAsc = meshDistributor->getPerDofAssociations(globalRowDof);
meshDistributor->getPerDofAssociations(globalRowDof); for (std::set<int>::iterator it = perRowAsc.begin();
for (std::set<int>::iterator it = perRowAsc.begin(); it != perRowAsc.end(); ++it)
it != perRowAsc.end(); ++it) if (*it >= -3)
if (*it >= -3) perAsc.insert(*it);
perAsc.insert(*it);
}
double scaledValue = double scaledValue =
value(*icursor) * std::pow(0.5, static_cast<double>(perAsc.size())); value(*icursor) * std::pow(0.5, static_cast<double>(perAsc.size()));
...@@ -195,10 +202,23 @@ namespace AMDiS { ...@@ -195,10 +202,23 @@ namespace AMDiS {
eIt != entry.end(); ++eIt) { eIt != entry.end(); ++eIt) {
// Calculate petsc row index. // Calculate petsc row index.
int rowIndex = eIt->first * dispMult + dispAddRow; int rowIndex = eIt->first * dispMult + dispAddRow;
int colIndex = eIt->second * dispMult + dispAddCol; int colIndex = eIt->second * dispMult + dispAddCol;
MatSetValue(petscMatrix, rowIndex, colIndex, scaledValue, ADD_VALUES); colsMap[rowIndex].push_back(colIndex);
valsMap[rowIndex].push_back(scaledValue);
} }
} }
for (std::map<int, std::vector<int> >::iterator rowIt = colsMap.begin();
rowIt != colsMap.end(); ++rowIt) {
TEST_EXIT_DBG(rowIt->second.size() == valsMap[rowIt->first].size())
("Should not happen!\n");
int rowIndex = rowIt->first;
MatSetValues(petscMatrix, 1, &rowIndex, rowIt->second.size(),
&(rowIt->second[0]), &(valsMap[rowIt->first][0]), ADD_VALUES);
}
} }
} }
} }
...@@ -256,111 +276,123 @@ namespace AMDiS { ...@@ -256,111 +276,123 @@ namespace AMDiS {
namespace traits = mtl::traits; namespace traits = mtl::traits;
typedef DOFMatrix::base_matrix_type Matrix; typedef DOFMatrix::base_matrix_type Matrix;
typedef std::vector<std::pair<int, int> > MatrixNnzEntry; typedef std::vector<std::pair<int, int> > MatrixNnzEntry;
typedef std::map<int, DofContainer> RankToDofContainer;
// Stores to each rank a list of nnz entries (i.e. pairs of row and column index) // Stores to each rank a list of nnz entries (i.e. pairs of row and column index)
// that this rank will send to. This nnz entries will be assembled on this rank, // that this rank will send to. These nnz entries will be assembled on this rank,
// but because the row DOFs are not DOFs of this rank they will be send to the // but because the row DOFs are not DOFs of this rank they will be send to the
// owner of the row DOFs. // owner of the row DOFs.
std::map<int, MatrixNnzEntry> sendMatrixEntry; std::map<int, MatrixNnzEntry> sendMatrixEntry;
// Maps to each DOF that must be send to another rank the rank number of the
// receiving rank.
std::map<DegreeOfFreedom, int> sendDofToRank;
// First, create for all ranks we send data to MatrixNnzEntry object with 0 entries. // First, create for all ranks we send data to MatrixNnzEntry object with 0 entries.
typedef std::map<int, DofContainer> RankToDofContainer; for (RankToDofContainer::iterator it = meshDistributor->getRecvDofs().begin();
RankToDofContainer& recvDofs = meshDistributor->getRecvDofs(); it != meshDistributor->getRecvDofs().end(); ++it) {
for (RankToDofContainer::iterator it = recvDofs.begin();
it != recvDofs.end(); ++it)
sendMatrixEntry[it->first].resize(0); sendMatrixEntry[it->first].resize(0);
for (DofContainer::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt)
sendDofToRank[**dofIt] = it->first;
}
std::set<int> recvFromRank;
for (RankToDofContainer::iterator it = meshDistributor->getSendDofs().begin();
it != meshDistributor->getSendDofs().end(); ++it)
recvFromRank.insert(it->first);
for (int i = 0; i < nComponents; i++) { for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++) { for (int j = 0; j < nComponents; j++) {
if ((*mat)[i][j]) { if (!(*mat)[i][j])
Matrix bmat = (*mat)[i][j]->getBaseMatrix(); continue;
Matrix bmat = (*mat)[i][j]->getBaseMatrix();
traits::col<Matrix>::type col(bmat); traits::col<Matrix>::type col(bmat);
traits::const_value<Matrix>::type value(bmat); traits::const_value<Matrix>::type value(bmat);
typedef traits::range_generator<row, Matrix>::type cursor_type; typedef traits::range_generator<row, Matrix>::type cursor_type;
typedef traits::range_generator<nz, cursor_type>::type icursor_type; typedef traits::range_generator<nz, cursor_type>::type icursor_type;
for (cursor_type cursor = begin<row>(bmat),
cend = end<row>(bmat); cursor != cend; ++cursor) {
for (cursor_type cursor = begin<row>(bmat), int globalRowDof = meshDistributor->mapLocalToGlobal(*cursor);
cend = end<row>(bmat); cursor != cend; ++cursor) {
// Map the local row number to the global DOF index and create from it
// the global PETSc row index of this DOF.
int petscRowIdx =
meshDistributor->mapLocalToGlobal(*cursor) * nComponents + i;
if (meshDistributor->getIsRankDof(*cursor)) {
// === The current row DOF is a rank dof, so create the corresponding === std::vector<int> rows;
// === nnz values directly on rank's nnz data. === rows.push_back(globalRowDof);
std::vector<int> rowRank;
// This is the local row index of the local PETSc matrix. if (meshDistributor->getIsRankDof(*cursor)) {
int localPetscRowIdx = rowRank.push_back(meshDistributor->getMpiRank());
petscRowIdx - meshDistributor->getRstart() * nComponents; } else {
// Find out who is the member of this DOF.
TEST_EXIT_DBG(sendDofToRank.count(*cursor))("Should not happen!\n");
rowRank.push_back(sendDofToRank[*cursor]);
}
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows) // Map the local row number to the global DOF index and create from it
("Should not happen! \n Debug info: localRowIdx = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d nCompontens = %d nRankRows = %d\n", // the global PETSc row index of this DOF.
*cursor, meshDistributor->mapLocalToGlobal(*cursor), petscRowIdx, localPetscRowIdx, meshDistributor->getRstart(), nComponents, nRankRows);
int petscRowIdx = globalRowDof * nComponents + i;
if (meshDistributor->getIsRankDof(*cursor)) {
// === The current row DOF is a rank dof, so create the corresponding ===
// === nnz values directly on rank's nnz data. ===
// This is the local row index of the local PETSc matrix.
int localPetscRowIdx =
petscRowIdx - meshDistributor->getRstart() * nComponents;
TEST_EXIT_DBG(localPetscRowIdx >= 0 && localPetscRowIdx < nRankRows)
("Should not happen! \n Debug info: localRowIdx = %d globalRowIndx = %d petscRowIdx = %d localPetscRowIdx = %d rStart = %d nCompontens = %d nRankRows = %d\n",
*cursor, meshDistributor->mapLocalToGlobal(*cursor), petscRowIdx, localPetscRowIdx, meshDistributor->getRstart(), nComponents, nRankRows);
// Traverse all non zero entries in this row.
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
// Traverse all non zero entries in this row. if (value(*icursor) != 0.0 || petscRowIdx == petscColIdx) {
for (icursor_type icursor = begin<nz>(cursor), // The row DOF is a rank DOF, if also the column is a rank DOF,
icend = end<nz>(cursor); icursor != icend; ++icursor) { // increment the d_nnz values for this row, otherwise the o_nnz value.
if (petscColIdx >= meshDistributor->getRstart() * nComponents &&
petscColIdx < meshDistributor->getRstart() * nComponents + nRankRows)
d_nnz[localPetscRowIdx]++;
else
o_nnz[localPetscRowIdx]++;
}
}
} else {
// === The current row DOF is not a rank dof, i.e., it will be created ===
// === on this rank, but after this it will be send to another rank ===
// === matrix. So we need to send also the corresponding nnz structure ===
// === of this row to the corresponding rank. ===
// Send all non zero entries to the member of the row DOF.
int sendToRank = sendDofToRank[*cursor];
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (value(*icursor) != 0.0) {
int petscColIdx = int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j; meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
if (value(*icursor) != 0.0 || petscRowIdx == petscColIdx) { sendMatrixEntry[sendToRank].
// The row DOF is a rank DOF, if also the column is a rank DOF, push_back(std::make_pair(petscRowIdx, petscColIdx));
// increment the d_nnz values for this row, otherwise the o_nnz value.
if (petscColIdx >= meshDistributor->getRstart() * nComponents &&
petscColIdx < meshDistributor->getRstart() * nComponents + nRankRows)
d_nnz[localPetscRowIdx]++;
else
o_nnz[localPetscRowIdx]++;
}
}
} else {
typedef std::map<int, DofContainer> RankToDofContainer;
// === The current row DOF is not a rank dof, i.e., it will be created ===
// === on this rank, but after this it will be send to another rank ===
// === matrix. So we need to send also the corresponding nnz structure ===
// === of this row to the corresponding rank. ===
// Find out who is the member of this DOF.
int sendToRank = -1;
for (RankToDofContainer::iterator it = meshDistributor->getRecvDofs().begin();
it != meshDistributor->getRecvDofs().end(); ++it) {
for (DofContainer::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (**dofIt == *cursor) {
sendToRank = it->first;
break;
}
}
if (sendToRank != -1)
break;
}
TEST_EXIT_DBG(sendToRank != -1)("Should not happen!\n");
// Send all non zero entries to the member of the row DOF.
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (value(*icursor) != 0.0) {
int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
sendMatrixEntry[sendToRank].
push_back(std::make_pair(petscRowIdx, petscColIdx));
}
} }
}
} // if (isRankDof[*cursor]) ... else ...
} // for each row in mat[i][j] } // if (isRankDof[*cursor]) ... else ...
} // if mat[i][j] } // for each row in mat[i][j]
} }
} }
...@@ -368,7 +400,9 @@ namespace AMDiS { ...@@ -368,7 +400,9 @@ namespace AMDiS {
StdMpi<MatrixNnzEntry> stdMpi(meshDistributor->getMpiComm(), true); StdMpi<MatrixNnzEntry> stdMpi(meshDistributor->getMpiComm(), true);
stdMpi.send(sendMatrixEntry); stdMpi.send(sendMatrixEntry);
stdMpi.recv(meshDistributor->getSendDofs()); for (std::set<int>::iterator it = recvFromRank.begin();
it != recvFromRank.end(); ++it)
stdMpi.recv(*it);
stdMpi.startCommunication<int>(MPI_INT); stdMpi.startCommunication<int>(MPI_INT);
...@@ -472,8 +506,8 @@ namespace AMDiS { ...@@ -472,8 +506,8 @@ namespace AMDiS {
for (int i = 0; i < nComponents; i++) for (int i = 0; i < nComponents; i++)
for (int j = 0; j < nComponents; j++) for (int j = 0; j < nComponents; j++)
if ((*mat)[i][j]) if ((*mat)[i][j])
setDofMatrix((*mat)[i][j], nComponents, i, j); setDofMatrix((*mat)[i][j], nComponents, i, j);
#if (DEBUG != 0) #if (DEBUG != 0)
INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", MPI::Wtime() - wtime); INFO(info, 8)("Fill petsc matrix 2 needed %.5f seconds\n", MPI::Wtime() - wtime);
#endif #endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment