Commit dfa8ce49 authored by Thomas Witkowski's avatar Thomas Witkowski

Fixed PETSc nnz problem.

parent 4f18875c
...@@ -44,7 +44,7 @@ available_tags=" CXX F77" ...@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG # ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host deimos104: # Libtool was configured on host p1d098:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -82,13 +82,13 @@ AR="ar" ...@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O3" LTCFLAGS="-g -O3"
# A language-specific compiler. # A language-specific compiler.
CC="gcc" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -174,7 +174,7 @@ dlopen_self=unknown ...@@ -174,7 +174,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6763,7 +6763,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac` ...@@ -6763,7 +6763,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End: # End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX # ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host deimos104: # Libtool was configured on host p1d098:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -6801,13 +6801,13 @@ AR="ar" ...@@ -6801,13 +6801,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O3" LTCFLAGS="-g -O3"
# A language-specific compiler. # A language-specific compiler.
CC="g++" CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -6893,7 +6893,7 @@ dlopen_self=unknown ...@@ -6893,7 +6893,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6960,11 +6960,11 @@ predeps="" ...@@ -6960,11 +6960,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a # Dependencies to place after the objects being linked to create a
# shared library. # shared library.
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s" postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking # The library search path used internally by the compiler when linking
# a shared library. # a shared library.
compiler_lib_search_path=`echo "-L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"` compiler_lib_search_path=`echo "-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
# Method to check whether dependent libraries are shared objects. # Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all" deplibs_check_method="pass_all"
...@@ -7071,7 +7071,7 @@ include_expsyms="" ...@@ -7071,7 +7071,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77 # ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host deimos104: # Libtool was configured on host p1d098:
# Shell to use when invoking shell scripts. # Shell to use when invoking shell scripts.
SHELL="/bin/sh" SHELL="/bin/sh"
...@@ -7109,7 +7109,7 @@ AR="ar" ...@@ -7109,7 +7109,7 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O3" LTCFLAGS="-g -O3"
......
...@@ -24,10 +24,12 @@ ...@@ -24,10 +24,12 @@
namespace AMDiS { namespace AMDiS {
using namespace std;
PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *) PetscErrorCode myKSPMonitor(KSP ksp, PetscInt iter, PetscReal rnorm, void *)
{ {
if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0) if (iter % 100 == 0 && MPI::COMM_WORLD.Get_rank() == 0)
std::cout << "[0] Petsc-Iteration " << iter << ": " << rnorm << std::endl; cout << "[0] Petsc-Iteration " << iter << ": " << rnorm << endl;
return 0; return 0;
} }
...@@ -66,12 +68,12 @@ namespace AMDiS { ...@@ -66,12 +68,12 @@ namespace AMDiS {
typedef traits::range_generator<row, Matrix>::type cursor_type; typedef traits::range_generator<row, Matrix>::type cursor_type;
typedef traits::range_generator<nz, cursor_type>::type icursor_type; typedef traits::range_generator<nz, cursor_type>::type icursor_type;
std::vector<int> cols; vector<int> cols;
std::vector<double> values; vector<double> values;
cols.reserve(300); cols.reserve(300);
values.reserve(300); values.reserve(300);
std::vector<int> globalCols; vector<int> globalCols;
// === Traverse all rows of the dof matrix and insert row wise the values === // === Traverse all rows of the dof matrix and insert row wise the values ===
...@@ -86,7 +88,9 @@ namespace AMDiS { ...@@ -86,7 +88,9 @@ namespace AMDiS {
bool periodicRow = meshDistributor->isPeriodicDof(globalRowDof); bool periodicRow = meshDistributor->isPeriodicDof(globalRowDof);
if (!periodicRow) { if (!periodicRow) {
// Calculate petsc row index. // === Row DOF index is not periodic. ===
// Calculate PETSc row index.
int rowIndex = globalRowDof * dispMult + dispAddRow; int rowIndex = globalRowDof * dispMult + dispAddRow;
cols.clear(); cols.clear();
...@@ -99,35 +103,54 @@ namespace AMDiS { ...@@ -99,35 +103,54 @@ namespace AMDiS {
int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor)); int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor));
// Test if the current col dof is a periodic dof. // Test if the current col dof is a periodic dof.
bool periodicCol = meshDistributor->isPeriodicDof(globalColDof); bool periodicCol = meshDistributor->isPeriodicDof(globalColDof);
// Calculate PETSc col index.
int colIndex = globalColDof * dispMult + dispAddCol;
if (value(*icursor) == 0.0 && globalRowDof != globalColDof) // Ignore all zero entries, expect it is a diagonal entry.
// if (value(*icursor) == 0.0 && globalRowDof != globalColDof)
if (value(*icursor) == 0.0 && rowIndex != colIndex)
continue; continue;
if (!periodicCol) { if (!periodicCol) {
// Calculate the exact position of the column index in the petsc matrix. // Calculate the exact position of the column index in the PETSc matrix.
cols.push_back(globalColDof * dispMult + dispAddCol); cols.push_back(globalColDof * dispMult + dispAddCol);
values.push_back(value(*icursor)); values.push_back(value(*icursor));
} else { } else {
std::set<int>& perColAsc = // === Row index is not periodic, but column index is. ===
meshDistributor->getPerDofAssociations(globalColDof);
// Create set of all periodic associations of the column index.
std::set<int> perAsc; std::set<int> perAsc;
std::set<int>& perColAsc =
meshDistributor->getPerDofAssociations(globalColDof);
for (std::set<int>::iterator it = perColAsc.begin(); for (std::set<int>::iterator it = perColAsc.begin();
it != perColAsc.end(); ++it) it != perColAsc.end(); ++it)
if (*it >= -3) if (*it >= -3)
perAsc.insert(*it); perAsc.insert(*it);
// Scale value to the number of periodic associations of the column index.
double scaledValue = double scaledValue =
value(*icursor) * std::pow(0.5, static_cast<double>(perAsc.size())); value(*icursor) * pow(0.5, static_cast<double>(perAsc.size()));
std::vector<int> newCols;
// === Create set of all matrix column indices due to the periodic ===
// === associations of the column DOF index. ===
vector<int> newCols;
// First, add the original matrix index.
newCols.push_back(globalColDof); newCols.push_back(globalColDof);
// And add all periodic matrix indices.
for (std::set<int>::iterator it = perAsc.begin(); for (std::set<int>::iterator it = perAsc.begin();
it != perAsc.end(); ++it) { it != perAsc.end(); ++it) {
int nCols = static_cast<int>(newCols.size()); for (unsigned int i = 0; i < newCols.size(); i++) {
for (int i = 0; i < nCols; i++) {
TEST_EXIT(meshDistributor->isPeriodicDof(newCols[i], *it)) TEST_EXIT(meshDistributor->isPeriodicDof(newCols[i], *it))
("Should not happen: %d %d\n", *it, newCols[i]); ("Wrong periodic DOF associations at boundary %d with DOF %d!\n",
*it, newCols[i]);
MSG("MAP DOF %d -> %d\n",
newCols[i],
meshDistributor->getPeriodicMapping(newCols[i], *it));
newCols.push_back(meshDistributor->getPeriodicMapping(newCols[i], *it)); newCols.push_back(meshDistributor->getPeriodicMapping(newCols[i], *it));
} }
...@@ -143,18 +166,30 @@ namespace AMDiS { ...@@ -143,18 +166,30 @@ namespace AMDiS {
MatSetValues(petscMatrix, 1, &rowIndex, cols.size(), MatSetValues(petscMatrix, 1, &rowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES); &(cols[0]), &(values[0]), ADD_VALUES);
} else { } else {
std::map<int, std::vector<int> > colsMap; // === Row DOF index is periodic. ===
std::map<int, std::vector<double> > valsMap;
// Because this row is periodic, we will have to add the entries of this
// matrix row to multiple rows. The following maps store to each row an
// array of column indices and values of the entries that must be added to
// the PETSc matrix.
map<int, vector<int> > colsMap;
map<int, vector<double> > valsMap;
// Traverse all column entries.
for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor);
icursor != icend; ++icursor) { icursor != icend; ++icursor) {
// Global index of the current column index. // Global index of the current column index.
int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor)); int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor));
// Ignore all zero entries, expect it is a diagonal entry.
if (value(*icursor) == 0.0 && globalRowDof != globalColDof) if (value(*icursor) == 0.0 && globalRowDof != globalColDof)
continue; continue;
// === Add all periodic associations of both, the row and the column ===
// === indices to the set perAsc. ===
std::set<int> perAsc; std::set<int> perAsc;
if (meshDistributor->isPeriodicDof(globalColDof)) { if (meshDistributor->isPeriodicDof(globalColDof)) {
...@@ -173,11 +208,21 @@ namespace AMDiS { ...@@ -173,11 +208,21 @@ namespace AMDiS {
if (*it >= -3) if (*it >= -3)
perAsc.insert(*it); perAsc.insert(*it);
// Scale the value with respect to the number of periodic associations.
double scaledValue = double scaledValue =
value(*icursor) * std::pow(0.5, static_cast<double>(perAsc.size())); value(*icursor) * pow(0.5, static_cast<double>(perAsc.size()));
std::vector<std::pair<int, int> > entry;
entry.push_back(std::make_pair(globalRowDof, globalColDof));
// === Create all matrix entries with respect to the periodic ===
// === associations of the row and column indices. ===
vector<pair<int, int> > entry;
// First, add the original entry.
entry.push_back(make_pair(globalRowDof, globalColDof));
// Then, traverse the periodic associations of the row and column indices
// and create the corresponding entries.
for (std::set<int>::iterator it = perAsc.begin(); it != perAsc.end(); ++it) { for (std::set<int>::iterator it = perAsc.begin(); it != perAsc.end(); ++it) {
int nEntry = static_cast<int>(entry.size()); int nEntry = static_cast<int>(entry.size());
for (int i = 0; i < nEntry; i++) { for (int i = 0; i < nEntry; i++) {
...@@ -194,23 +239,28 @@ namespace AMDiS { ...@@ -194,23 +239,28 @@ namespace AMDiS {
perColDof = entry[i].second; perColDof = entry[i].second;
entry.push_back(std::make_pair(perRowDof, perColDof)); entry.push_back(make_pair(perRowDof, perColDof));
} }
} }
for (std::vector<std::pair<int, int> >::iterator eIt = entry.begin();
// === Translate the matrix entries to PETSc's matrix.
for (vector<pair<int, int> >::iterator eIt = entry.begin();
eIt != entry.end(); ++eIt) { eIt != entry.end(); ++eIt) {
// Calculate petsc row index. // Calculate row and column indices of the PETSc matrix.
int rowIndex = eIt->first * dispMult + dispAddRow; int rowIndex = eIt->first * dispMult + dispAddRow;
int colIndex = eIt->second * dispMult + dispAddCol; int colIndex = eIt->second * dispMult + dispAddCol;
colsMap[rowIndex].push_back(colIndex); colsMap[rowIndex].push_back(colIndex);
valsMap[rowIndex].push_back(scaledValue); valsMap[rowIndex].push_back(scaledValue);
} }
} }
for (std::map<int, std::vector<int> >::iterator rowIt = colsMap.begin(); // === Finally, add all periodic rows to the PETSc matrix. ===
for (map<int, vector<int> >::iterator rowIt = colsMap.begin();
rowIt != colsMap.end(); ++rowIt) { rowIt != colsMap.end(); ++rowIt) {
TEST_EXIT_DBG(rowIt->second.size() == valsMap[rowIt->first].size()) TEST_EXIT_DBG(rowIt->second.size() == valsMap[rowIt->first].size())
("Should not happen!\n"); ("Should not happen!\n");
...@@ -275,18 +325,18 @@ namespace AMDiS { ...@@ -275,18 +325,18 @@ namespace AMDiS {
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end; using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits; namespace traits = mtl::traits;
typedef DOFMatrix::base_matrix_type Matrix; typedef DOFMatrix::base_matrix_type Matrix;
typedef std::vector<std::pair<int, int> > MatrixNnzEntry; typedef vector<pair<int, int> > MatrixNnzEntry;
typedef std::map<int, DofContainer> RankToDofContainer; typedef map<int, DofContainer> RankToDofContainer;
// Stores to each rank a list of nnz entries (i.e. pairs of row and column index) // Stores to each rank a list of nnz entries (i.e. pairs of row and column index)
// that this rank will send to. These nnz entries will be assembled on this rank, // that this rank will send to. These nnz entries will be assembled on this rank,
// but because the row DOFs are not DOFs of this rank they will be send to the // but because the row DOFs are not DOFs of this rank they will be send to the
// owner of the row DOFs. // owner of the row DOFs.
std::map<int, MatrixNnzEntry> sendMatrixEntry; map<int, MatrixNnzEntry> sendMatrixEntry;
// Maps to each DOF that must be send to another rank the rank number of the // Maps to each DOF that must be send to another rank the rank number of the
// receiving rank. // receiving rank.
std::map<DegreeOfFreedom, int> sendDofToRank; map<DegreeOfFreedom, int> sendDofToRank;
// First, create for all ranks we send data to MatrixNnzEntry object with 0 entries. // First, create for all ranks we send data to MatrixNnzEntry object with 0 entries.
...@@ -324,9 +374,9 @@ namespace AMDiS { ...@@ -324,9 +374,9 @@ namespace AMDiS {
int globalRowDof = meshDistributor->mapLocalToGlobal(*cursor); int globalRowDof = meshDistributor->mapLocalToGlobal(*cursor);
std::vector<int> rows; vector<int> rows;
rows.push_back(globalRowDof); rows.push_back(globalRowDof);
std::vector<int> rowRank; vector<int> rowRank;
if (meshDistributor->getIsRankDof(*cursor)) { if (meshDistributor->getIsRankDof(*cursor)) {
rowRank.push_back(meshDistributor->getMpiRank()); rowRank.push_back(meshDistributor->getMpiRank());
} else { } else {
...@@ -387,7 +437,7 @@ namespace AMDiS { ...@@ -387,7 +437,7 @@ namespace AMDiS {
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j; meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
sendMatrixEntry[sendToRank]. sendMatrixEntry[sendToRank].
push_back(std::make_pair(petscRowIdx, petscColIdx)); push_back(make_pair(petscRowIdx, petscColIdx));
} }
} }
...@@ -409,7 +459,7 @@ namespace AMDiS { ...@@ -409,7 +459,7 @@ namespace AMDiS {
// === Evaluate the nnz structure this rank got from other ranks and add it to === // === Evaluate the nnz structure this rank got from other ranks and add it to ===
// === the PETSc nnz data structure. === // === the PETSc nnz data structure. ===
for (std::map<int, MatrixNnzEntry>::iterator it = stdMpi.getRecvData().begin(); for (map<int, MatrixNnzEntry>::iterator it = stdMpi.getRecvData().begin();
it != stdMpi.getRecvData().end(); ++it) { it != stdMpi.getRecvData().end(); ++it) {
if (it->second.size() > 0) { if (it->second.size() > 0) {
for (unsigned int i = 0; i < it->second.size(); i++) { for (unsigned int i = 0; i < it->second.size(); i++) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment