Skip to content

GitLab

Commit c541effa authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

Several changes in parallel AMDiS interface.

parent e913e5aa
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 365 additions and 385 deletions
AMDiS/libtool
View file @ c541effa
......@@ -30,10 +30,10 @@
# the same distribution terms that you use for the rest of that program.
# A sed program that does not truncate output.
SED="/bin/sed"
SED="/usr/bin/sed"
# Sed that helps us avoid accidentally triggering echo(1) options like -n.
Xsed="/bin/sed -e 1s/^X//"
Xsed="/usr/bin/sed -e 1s/^X//"
# The HP-UX ksh and POSIX shell print the target directory to stdout
# if CDPATH is set.
......@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host NWRW13:
# Libtool was configured on host p1d066:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -66,12 +66,12 @@ fast_install=yes
# The host system.
host_alias=
host=i686-pc-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-pc-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
......@@ -82,25 +82,22 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
LTCC="gcc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
CC="gcc"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU C compiler?
with_gcc=yes
gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
gcc_ver=`gcc -dumpversion`
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
......@@ -174,7 +171,7 @@ dlopen_self=unknown
dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking.
link_static_flag="-static"
link_static_flag=""
# Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin"
......@@ -232,11 +229,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
predep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
predep_objects=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
postdep_objects=""
# Dependencies to place before the objects being linked to create a
# shared library.
......@@ -248,7 +245,7 @@ postdeps=""
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
compiler_lib_search_path=""
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
......@@ -328,10 +325,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
sys_lib_search_path_spec=`echo " /u/backofen/adds/local/lib/i386-redhat-linux/4.1.2/ /u/backofen/adds/local/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/4.1.2/ /fastfs/wir/local/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /fastfs/wir/local/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
......@@ -6763,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host NWRW13:
# Libtool was configured on host p1d066:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -6785,12 +6782,12 @@ fast_install=yes
# The host system.
host_alias=
host=i686-pc-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-pc-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
......@@ -6801,25 +6798,22 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
LTCC="gcc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
CC="g++"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpiCC"
# Is the compiler the GNU C compiler?
with_gcc=yes
gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
gcc_ver=`gcc -dumpversion`
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
......@@ -6893,7 +6887,7 @@ dlopen_self=unknown
dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking.
link_static_flag="-static"
link_static_flag=""
# Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin"
......@@ -6948,11 +6942,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
predep_objects=`echo "/usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crti.o /usr/lib/gcc/i386-redhat-linux/4.1.2/crtbeginS.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
# Dependencies to place after the objects being linked to create a
# shared library.
postdep_objects=`echo "/usr/lib/gcc/i386-redhat-linux/4.1.2/crtendS.o /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crtn.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
# Dependencies to place before the objects being linked to create a
# shared library.
......@@ -6960,11 +6954,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path=`echo "-L/u/backofen/adds/local/lib -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
......@@ -7044,10 +7038,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
sys_lib_search_path_spec=`echo " /u/backofen/adds/local/lib/i386-redhat-linux/4.1.2/ /u/backofen/adds/local/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/4.1.2/ /fastfs/wir/local/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /fastfs/wir/local/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
......@@ -7071,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host NWRW13:
# Libtool was configured on host p1d066:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -7093,12 +7087,12 @@ fast_install=yes
# The host system.
host_alias=
host=i686-pc-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-pc-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
......@@ -7109,25 +7103,22 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
LTCC="gcc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
CC="f95"
CC="g77"
# Is the compiler the GNU C compiler?
with_gcc=yes
gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
gcc_ver=`gcc -dumpversion`
with_gcc=
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
......@@ -7259,11 +7250,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
predep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
predep_objects=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
postdep_objects=""
# Dependencies to place before the objects being linked to create a
# shared library.
......@@ -7275,7 +7266,7 @@ postdeps=""
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
compiler_lib_search_path=""
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
......@@ -7355,10 +7346,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
sys_lib_search_path_spec=`echo " /u/backofen/adds/local/lib/i386-redhat-linux/4.1.2/ /u/backofen/adds/local/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/3.3.5/ /fastfs/wir/local/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/ /usr/lib/gcc/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../ /lib/x86_64-suse-linux/3.3.5/ /lib/ /usr/lib/x86_64-suse-linux/3.3.5/ /usr/lib/"
# Run-time system search path for libraries
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
......
AMDiS/src/parallel/GlobalMatrixSolver.cc
View file @ c541effa
......@@ -16,17 +16,36 @@ namespace AMDiS {
return 0;
}
void GlobalMatrixSolver::solve()
void GlobalMatrixSolver::addToMeshDistributor(MeshDistributor& m)
{
meshDistributor = &m;
m.addProblemStat(this);
}
void GlobalMatrixSolver::buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
bool assembleVector)
{
meshDistributor->checkMeshChange();
ProblemVec::buildAfterCoarsen(adaptInfo, flag, assembleMatrix, assembleVector);
}
void GlobalMatrixSolver::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
{
FUNCNAME("GlobalMatrixSolver::solve()");
TEST_EXIT(meshDistributor)("Should not happen!\n");
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock();
fillPetscMatrix(probStat->getSystemMatrix(), probStat->getRhs());
solvePetscMatrix(*(probStat->getSolution()));
fillPetscMatrix(systemMatrix, rhs);
solvePetscMatrix(*solution);
#ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
......@@ -71,9 +90,9 @@ namespace AMDiS {
values.clear();
// Global index of the current row dof.
int globalRowDof = mapLocalGlobalDofs[*cursor];
DegreeOfFreedom globalRowDof = meshDistributor->mapLocalToGlobal(*cursor);
// Test if the current row dof is a periodic dof.
bool periodicRow = (periodicDof.count(globalRowDof) > 0);
bool periodicRow = (meshDistributor->getPeriodicDofMap().count(globalRowDof) > 0);
// === Traverse all non zero entries of the row and produce vector cols ===
......@@ -86,16 +105,18 @@ namespace AMDiS {
// Set only non null values.
if (value(*icursor) != 0.0) {
// Global index of the current column index.
int globalColDof = mapLocalGlobalDofs[col(*icursor)];
int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor));
// Calculate the exact position of the column index in the petsc matrix.
int colIndex = globalColDof * dispMult + dispAddCol;
// If the current row is not periodic, but the current dof index is periodic,
// we have to duplicate the value to the other corresponding periodic columns.
if (periodicRow == false && periodicDof.count(globalColDof) > 0) {
if (periodicRow == false &&
meshDistributor->getPeriodicDofMap().count(globalColDof) > 0) {
// The value is assign to n matrix entries, therefore, every entry
// has only 1/n value of the original entry.
double scalFactor = 1.0 / (periodicDof[globalColDof].size() + 1.0);
double scalFactor =
1.0 / (meshDistributor->getPeriodicDof(globalColDof).size() + 1.0);
// Insert original entry.
cols.push_back(colIndex);
......@@ -103,8 +124,8 @@ namespace AMDiS {
// Insert the periodic entries.
for (std::set<DegreeOfFreedom>::iterator it =
periodicDof[globalColDof].begin();
it != periodicDof[globalColDof].end(); ++it) {
meshDistributor->getPeriodicDof(globalColDof).begin();
it != meshDistributor->getPeriodicDof(globalColDof).end(); ++it) {
cols.push_back(*it * dispMult + dispAddCol);
values.push_back(value(*icursor) * scalFactor);
}
......@@ -126,7 +147,8 @@ namespace AMDiS {
if (periodicRow) {
// The row dof is periodic, so send dof to all the corresponding rows.
double scalFactor = 1.0 / (periodicDof[globalRowDof].size() + 1.0);
double scalFactor =
1.0 / (meshDistributor->getPeriodicDof(globalRowDof).size() + 1.0);
int diagIndex = -1;
for (int i = 0; i < static_cast<int>(values.size()); i++) {
......@@ -149,8 +171,8 @@ namespace AMDiS {
values[diagIndex] = 0.0;
// Send the row to all periodic row indices.
for (std::set<DegreeOfFreedom>::iterator it = periodicDof[globalRowDof].begin();
it != periodicDof[globalRowDof].end(); ++it) {
for (std::set<DegreeOfFreedom>::iterator it = meshDistributor->getPeriodicDof(globalRowDof).begin();
it != meshDistributor->getPeriodicDof(globalRowDof).end(); ++it) {
int perRowIndex = *it * dispMult + dispAddRow;
MatSetValues(petscMatrix, 1, &perRowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES);
......@@ -172,18 +194,18 @@ namespace AMDiS {
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
for (dofIt.reset(); !dofIt.end(); ++dofIt) {
// Calculate global row index of the dof.
int globalRow = mapLocalGlobalDofs[dofIt.getDOFIndex()];
DegreeOfFreedom globalRow = meshDistributor->mapLocalToGlobal(dofIt.getDOFIndex());
// Calculate petsc index of the row dof.
int index = globalRow * dispMult + dispAdd;
if (periodicDof.count(globalRow) > 0) {
if (meshDistributor->getPeriodicDofMap().count(globalRow) > 0) {
// The dof index is periodic, so devide the value to all dof entries.
double value = *dofIt / (periodicDof[globalRow].size() + 1.0);
double value = *dofIt / (meshDistributor->getPeriodicDof(globalRow).size() + 1.0);
VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
for (std::set<DegreeOfFreedom>::iterator it = periodicDof[globalRow].begin();
it != periodicDof[globalRow].end(); ++it) {
for (std::set<DegreeOfFreedom>::iterator it = meshDistributor->getPeriodicDof(globalRow).begin();
it != meshDistributor->getPeriodicDof(globalRow).end(); ++it) {
index = *it * dispMult + dispAdd;
VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
}
......@@ -204,6 +226,7 @@ namespace AMDiS {
TEST_EXIT_DBG(!d_nnz)("There is something wrong!\n");
TEST_EXIT_DBG(!o_nnz)("There is something wrong!\n");
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
d_nnz = new int[nRankRows];
o_nnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++) {
......@@ -238,21 +261,23 @@ namespace AMDiS {
// Map the local row number to the global DOF index and create from it
// the global PETSc row index of this DOF.
int petscRowIdx = mapLocalGlobalDofs[*cursor] * nComponents + i;
int petscRowIdx =
meshDistributor->mapLocalToGlobal(*cursor) * nComponents + i;
if (isRankDof[*cursor]) {
if (meshDistributor->getIsRankDof(*cursor)) {
// === The current row DOF is a rank dof, so create the corresponding ===
// === nnz values directly on rank's nnz data. ===
// This is the local row index of the local PETSc matrix.
int localPetscRowIdx = petscRowIdx - rstart * nComponents;
int localPetscRowIdx =
petscRowIdx - meshDistributor->getRstart() * nComponents;
#if (DEBUG != 0)
if (localPetscRowIdx < 0 || localPetscRowIdx >= nRankRows) {
std::cout << "ERROR in rank: " << mpiRank << std::endl;
std::cout << "ERROR in rank: " << meshDistributor->getMpiRank() << std::endl;
std::cout << " Wrong r = " << localPetscRowIdx << " " << *cursor
<< " " << mapLocalGlobalDofs[*cursor] << " "
<< " " << meshDistributor->mapLocalToGlobal(*cursor) << " "
<< nRankRows << std::endl;
ERROR_EXIT("Should not happen!\n");
}
......@@ -262,19 +287,21 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (value(*icursor) != 0.0) {
int petscColIdx = mapLocalGlobalDofs[col(*icursor)] * nComponents + j;
int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the d_nnz values for this row, otherwise the o_nnz value.
if (petscColIdx >= rstart * nComponents &&
petscColIdx < rstart * nComponents + nRankRows)
if (petscColIdx >= meshDistributor->getRstart() * nComponents &&
petscColIdx < meshDistributor->getRstart() * nComponents + nRankRows)
d_nnz[localPetscRowIdx]++;
else
o_nnz[localPetscRowIdx]++;
}
}
} else {
typedef std::map<int, DofContainer> RankToDofContainer;
// === The current row DOF is not a rank dof, i.e., it will be created ===
// === on this rank, but after this it will be send to another rank ===
// === matrix. So we need to send also the corresponding nnz structure ===
......@@ -282,17 +309,11 @@ namespace AMDiS {
// Find out who is the member of this DOF.
int sendToRank = -1;
for (RankToDofContainer::iterator it = recvDofs.begin();
it != recvDofs.end(); ++it) {
for (RankToDofContainer::iterator it = meshDistributor->getRecvDofs().begin();
it != meshDistributor->getRecvDofs().end(); ++it) {
for (DofContainer::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (**dofIt == *cursor) {
if (petscRowIdx == 6717) {
debug::writeDofMesh(mpiRank, *cursor, feSpace);
MSG("SEND DOF TO: %d/%d\n", it->first, *cursor);
}
sendToRank = it->first;
break;
}
......@@ -308,7 +329,8 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (value(*icursor) != 0.0) {
int petscColIdx = mapLocalGlobalDofs[col(*icursor)] * nComponents + j;
int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
sendMatrixEntry[sendToRank].
push_back(std::make_pair(petscRowIdx, petscColIdx));
......@@ -323,9 +345,9 @@ namespace AMDiS {
// === Send and recv the nnz row structure to/from other ranks. ===
StdMpi<MatrixNnzEntry> stdMpi(mpiComm, true);
StdMpi<MatrixNnzEntry> stdMpi(meshDistributor->getMpiComm(), true);
stdMpi.send(sendMatrixEntry);
stdMpi.recv(sendDofs);
stdMpi.recv(meshDistributor->getSendDofs());
stdMpi.startCommunication<int>(MPI_INT);
// === Evaluate the nnz structure this rank got from other ranks and add it to ===
......@@ -338,13 +360,14 @@ namespace AMDiS {
int r = it->second[i].first;
int c = it->second[i].second;
int localRowIdx = r - rstart * nComponents;
int localRowIdx = r - meshDistributor->getRstart() * nComponents;
TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows)
("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n",
r, localRowIdx, nRankRows, it->first);
if (c < rstart * nComponents || c >= rstart * nComponents + nRankRows)
if (c < meshDistributor->getRstart() * nComponents ||
c >= meshDistributor->getRstart() * nComponents + nRankRows)
o_nnz[localRowIdx]++;
else
d_nnz[localRowIdx]++;
......@@ -359,6 +382,8 @@ namespace AMDiS {
FUNCNAME("GlobalMatrixSolver::fillPetscMatrix()");
clock_t first = clock();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
// === Create PETSc vector (rhs, solution and a temporary vector). ===
......@@ -374,14 +399,14 @@ namespace AMDiS {
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
if (!d_nnz || lastMeshChangeIndex != lastMeshNnz) {
if (!d_nnz || meshDistributor->getLastMeshChangeIndex() != lastMeshNnz) {
if (d_nnz) {
delete [] d_nnz;
delete [] o_nnz;
}
createPetscNnzStructure(mat);
lastMeshNnz = lastMeshChangeIndex;
lastMeshNnz = meshDistributor->getLastMeshChangeIndex();
}
// === Create PETSc matrix with the computed nnz data structure. ===
......@@ -394,8 +419,10 @@ namespace AMDiS {
#if (DEBUG != 0)
int a, b;
MatGetOwnershipRange(petscMatrix, &a, &b);
TEST_EXIT(a == rstart * nComponents)("Wrong matrix ownership range!\n");
TEST_EXIT(b == rstart * nComponents + nRankRows)("Wrong matrix ownership range!\n");
TEST_EXIT(a == meshDistributor->getRstart() * nComponents)
("Wrong matrix ownership range!\n");
TEST_EXIT(b == meshDistributor->getRstart() * nComponents + nRankRows)
("Wrong matrix ownership range!\n");
#endif
// === Transfer values from DOF matrices to the PETSc matrix. ===
......@@ -456,10 +483,11 @@ namespace AMDiS {
PetscScalar *vecPointer;
VecGetArray(petscSolVec, &vecPointer);
int nRankDofs = meshDistributor->getNumberRankDofs();
for (int i = 0; i < nComponents; i++) {
DOFVector<double> *dofvec = vec.getDOFVector(i);
for (int j = 0; j < nRankDofs; j++)
(*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i];
(*dofvec)[meshDistributor->mapLocalToGlobal(j)] = vecPointer[j * nComponents + i];
}
VecRestoreArray(petscSolVec, &vecPointer);
......@@ -467,7 +495,7 @@ namespace AMDiS {
// === Synchronize dofs at common dofs, i.e., dofs that correspond to more ===
// === than one partition. ===
synchVector(vec);
meshDistributor->synchVector(vec);
// === Print information about solution process. ===
......
AMDiS/src/parallel/GlobalMatrixSolver.h
View file @ c541effa
......@@ -33,12 +33,14 @@
#include "petscao.h"
namespace AMDiS {
class GlobalMatrixSolver : public ParallelDomainBase
class GlobalMatrixSolver : public ProblemVec
{
public:
GlobalMatrixSolver(ProblemVec *problemStat, ProblemInstatVec *problemInstat)
: ParallelDomainBase(problemStat, problemInstat),
GlobalMatrixSolver(std::string nameStr,