Skip to content

GitLab

Commit c541effa authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Several changes in parallel AMDiS interface.

parent e913e5aa
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 365 additions and 385 deletions
AMDiS/libtool
View file @ c541effa
......@@ -30,10 +30,10 @@
# the same distribution terms that you use for the rest of that program.
# A sed program that does not truncate output.
SED="/bin/sed"
SED="/usr/bin/sed"
# Sed that helps us avoid accidentally triggering echo(1) options like -n.
Xsed="/bin/sed -e 1s/^X//"
Xsed="/usr/bin/sed -e 1s/^X//"
# The HP-UX ksh and POSIX shell print the target directory to stdout
# if CDPATH is set.
......@@ -44,7 +44,7 @@ available_tags=" CXX F77"
# ### BEGIN LIBTOOL CONFIG
# Libtool was configured on host NWRW13:
# Libtool was configured on host p1d066:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -66,12 +66,12 @@ fast_install=yes
# The host system.
host_alias=
host=i686-pc-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-pc-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
......@@ -82,25 +82,22 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
LTCC="gcc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
CC="gcc"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU C compiler?
with_gcc=yes
gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
gcc_ver=`gcc -dumpversion`
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
......@@ -174,7 +171,7 @@ dlopen_self=unknown
dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking.
link_static_flag="-static"
link_static_flag=""
# Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin"
......@@ -232,11 +229,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
predep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
predep_objects=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
postdep_objects=""
# Dependencies to place before the objects being linked to create a
# shared library.
......@@ -248,7 +245,7 @@ postdeps=""
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
compiler_lib_search_path=""
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
......@@ -328,10 +325,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
sys_lib_search_path_spec=`echo " /u/backofen/adds/local/lib/i386-redhat-linux/4.1.2/ /u/backofen/adds/local/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/4.1.2/ /fastfs/wir/local/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /fastfs/wir/local/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
......@@ -6763,7 +6760,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
# End:
# ### BEGIN LIBTOOL TAG CONFIG: CXX
# Libtool was configured on host NWRW13:
# Libtool was configured on host p1d066:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -6785,12 +6782,12 @@ fast_install=yes
# The host system.
host_alias=
host=i686-pc-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-pc-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
......@@ -6801,25 +6798,22 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
LTCC="gcc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
CC="g++"
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpiCC"
# Is the compiler the GNU C compiler?
with_gcc=yes
gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
gcc_ver=`gcc -dumpversion`
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
......@@ -6893,7 +6887,7 @@ dlopen_self=unknown
dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking.
link_static_flag="-static"
link_static_flag=""
# Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin"
......@@ -6948,11 +6942,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
predep_objects=`echo "/usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crti.o /usr/lib/gcc/i386-redhat-linux/4.1.2/crtbeginS.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
# Dependencies to place after the objects being linked to create a
# shared library.
postdep_objects=`echo "/usr/lib/gcc/i386-redhat-linux/4.1.2/crtendS.o /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crtn.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
# Dependencies to place before the objects being linked to create a
# shared library.
......@@ -6960,11 +6954,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path=`echo "-L/u/backofen/adds/local/lib -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
......@@ -7044,10 +7038,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
sys_lib_search_path_spec=`echo " /u/backofen/adds/local/lib/i386-redhat-linux/4.1.2/ /u/backofen/adds/local/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/4.1.2/ /fastfs/wir/local/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/ /usr/lib/gcc/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/../lib64/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../x86_64-suse-linux/4.1.2/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/ /lib/x86_64-suse-linux/4.1.2/ /lib/../lib64/ /usr/lib/x86_64-suse-linux/4.1.2/ /usr/lib/../lib64/ /fastfs/wir/local/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../ /lib/ /usr/lib/"
# Run-time system search path for libraries
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
......@@ -7071,7 +7065,7 @@ include_expsyms=""
# ### BEGIN LIBTOOL TAG CONFIG: F77
# Libtool was configured on host NWRW13:
# Libtool was configured on host p1d066:
# Shell to use when invoking shell scripts.
SHELL="/bin/sh"
......@@ -7093,12 +7087,12 @@ fast_install=yes
# The host system.
host_alias=
host=i686-pc-linux-gnu
host=x86_64-unknown-linux-gnu
host_os=linux-gnu
# The build system.
build_alias=
build=i686-pc-linux-gnu
build=x86_64-unknown-linux-gnu
build_os=linux-gnu
# An echo program that does not interpret backslashes.
......@@ -7109,25 +7103,22 @@ AR="ar"
AR_FLAGS="cru"
# A C compiler.
LTCC="gcc"
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
# A language-specific compiler.
CC="f95"
CC="g77"
# Is the compiler the GNU C compiler?
with_gcc=yes
gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
gcc_ver=`gcc -dumpversion`
with_gcc=
# An ERE matcher.
EGREP="grep -E"
# The linker used to build libraries.
LD="/usr/bin/ld"
LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
# Whether we need hard or soft links.
LN_S="ln -s"
......@@ -7259,11 +7250,11 @@ striplib="strip --strip-unneeded"
# Dependencies to place before the objects being linked to create a
# shared library.
predep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
predep_objects=""
# Dependencies to place after the objects being linked to create a
# shared library.
postdep_objects=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
postdep_objects=""
# Dependencies to place before the objects being linked to create a
# shared library.
......@@ -7275,7 +7266,7 @@ postdeps=""
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path=`echo "" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
compiler_lib_search_path=""
# Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all"
......@@ -7355,10 +7346,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
link_all_deplibs=unknown
# Compile-time system search path for libraries
sys_lib_search_path_spec=`echo " /u/backofen/adds/local/lib/i386-redhat-linux/4.1.2/ /u/backofen/adds/local/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
sys_lib_search_path_spec=" /fastfs/wir/local/lib/x86_64-suse-linux/3.3.5/ /fastfs/wir/local/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/ /usr/lib/gcc/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../../x86_64-suse-linux/lib/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../x86_64-suse-linux/3.3.5/ /usr/lib64/gcc-lib/x86_64-suse-linux/3.3.5/../../../ /lib/x86_64-suse-linux/3.3.5/ /lib/ /usr/lib/x86_64-suse-linux/3.3.5/ /usr/lib/"
# Run-time system search path for libraries
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/mysql /usr/lib/octave-3.0.1 /usr/lib/qt-3.3/lib /usr/lib/qt4/lib "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Fix the shell variable $srcfile for the compiler.
fix_srcfile_path=""
......
AMDiS/src/parallel/GlobalMatrixSolver.cc
View file @ c541effa
......@@ -16,17 +16,36 @@ namespace AMDiS {
return 0;
}
void GlobalMatrixSolver::solve()
void GlobalMatrixSolver::addToMeshDistributor(MeshDistributor& m)
{
meshDistributor = &m;
m.addProblemStat(this);
}
void GlobalMatrixSolver::buildAfterCoarsen(AdaptInfo *adaptInfo, Flag flag,
bool assembleMatrix,
bool assembleVector)
{
meshDistributor->checkMeshChange();
ProblemVec::buildAfterCoarsen(adaptInfo, flag, assembleMatrix, assembleVector);
}
void GlobalMatrixSolver::solve(AdaptInfo *adaptInfo, bool fixedMatrix)
{
FUNCNAME("GlobalMatrixSolver::solve()");
TEST_EXIT(meshDistributor)("Should not happen!\n");
#ifdef _OPENMP
double wtime = omp_get_wtime();
#endif
clock_t first = clock();
fillPetscMatrix(probStat->getSystemMatrix(), probStat->getRhs());
solvePetscMatrix(*(probStat->getSolution()));
fillPetscMatrix(systemMatrix, rhs);
solvePetscMatrix(*solution);
#ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
......@@ -71,9 +90,9 @@ namespace AMDiS {
values.clear();
// Global index of the current row dof.
int globalRowDof = mapLocalGlobalDofs[*cursor];
DegreeOfFreedom globalRowDof = meshDistributor->mapLocalToGlobal(*cursor);
// Test if the current row dof is a periodic dof.
bool periodicRow = (periodicDof.count(globalRowDof) > 0);
bool periodicRow = (meshDistributor->getPeriodicDofMap().count(globalRowDof) > 0);
// === Traverse all non zero entries of the row and produce vector cols ===
......@@ -86,16 +105,18 @@ namespace AMDiS {
// Set only non null values.
if (value(*icursor) != 0.0) {
// Global index of the current column index.
int globalColDof = mapLocalGlobalDofs[col(*icursor)];
int globalColDof = meshDistributor->mapLocalToGlobal(col(*icursor));
// Calculate the exact position of the column index in the petsc matrix.
int colIndex = globalColDof * dispMult + dispAddCol;
// If the current row is not periodic, but the current dof index is periodic,
// we have to duplicate the value to the other corresponding periodic columns.
if (periodicRow == false && periodicDof.count(globalColDof) > 0) {
if (periodicRow == false &&
meshDistributor->getPeriodicDofMap().count(globalColDof) > 0) {
// The value is assign to n matrix entries, therefore, every entry
// has only 1/n value of the original entry.
double scalFactor = 1.0 / (periodicDof[globalColDof].size() + 1.0);
double scalFactor =
1.0 / (meshDistributor->getPeriodicDof(globalColDof).size() + 1.0);
// Insert original entry.
cols.push_back(colIndex);
......@@ -103,8 +124,8 @@ namespace AMDiS {
// Insert the periodic entries.
for (std::set<DegreeOfFreedom>::iterator it =
periodicDof[globalColDof].begin();
it != periodicDof[globalColDof].end(); ++it) {
meshDistributor->getPeriodicDof(globalColDof).begin();
it != meshDistributor->getPeriodicDof(globalColDof).end(); ++it) {
cols.push_back(*it * dispMult + dispAddCol);
values.push_back(value(*icursor) * scalFactor);
}
......@@ -126,7 +147,8 @@ namespace AMDiS {
if (periodicRow) {
// The row dof is periodic, so send dof to all the corresponding rows.
double scalFactor = 1.0 / (periodicDof[globalRowDof].size() + 1.0);
double scalFactor =
1.0 / (meshDistributor->getPeriodicDof(globalRowDof).size() + 1.0);
int diagIndex = -1;
for (int i = 0; i < static_cast<int>(values.size()); i++) {
......@@ -149,8 +171,8 @@ namespace AMDiS {
values[diagIndex] = 0.0;
// Send the row to all periodic row indices.
for (std::set<DegreeOfFreedom>::iterator it = periodicDof[globalRowDof].begin();
it != periodicDof[globalRowDof].end(); ++it) {
for (std::set<DegreeOfFreedom>::iterator it = meshDistributor->getPeriodicDof(globalRowDof).begin();
it != meshDistributor->getPeriodicDof(globalRowDof).end(); ++it) {
int perRowIndex = *it * dispMult + dispAddRow;
MatSetValues(petscMatrix, 1, &perRowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES);
......@@ -172,18 +194,18 @@ namespace AMDiS {
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
for (dofIt.reset(); !dofIt.end(); ++dofIt) {
// Calculate global row index of the dof.
int globalRow = mapLocalGlobalDofs[dofIt.getDOFIndex()];
DegreeOfFreedom globalRow = meshDistributor->mapLocalToGlobal(dofIt.getDOFIndex());
// Calculate petsc index of the row dof.
int index = globalRow * dispMult + dispAdd;
if (periodicDof.count(globalRow) > 0) {
if (meshDistributor->getPeriodicDofMap().count(globalRow) > 0) {
// The dof index is periodic, so devide the value to all dof entries.
double value = *dofIt / (periodicDof[globalRow].size() + 1.0);
double value = *dofIt / (meshDistributor->getPeriodicDof(globalRow).size() + 1.0);
VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
for (std::set<DegreeOfFreedom>::iterator it = periodicDof[globalRow].begin();
it != periodicDof[globalRow].end(); ++it) {
for (std::set<DegreeOfFreedom>::iterator it = meshDistributor->getPeriodicDof(globalRow).begin();
it != meshDistributor->getPeriodicDof(globalRow).end(); ++it) {
index = *it * dispMult + dispAdd;
VecSetValues(petscVec, 1, &index, &value, ADD_VALUES);
}
......@@ -204,6 +226,7 @@ namespace AMDiS {
TEST_EXIT_DBG(!d_nnz)("There is something wrong!\n");
TEST_EXIT_DBG(!o_nnz)("There is something wrong!\n");
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
d_nnz = new int[nRankRows];
o_nnz = new int[nRankRows];
for (int i = 0; i < nRankRows; i++) {
......@@ -238,21 +261,23 @@ namespace AMDiS {
// Map the local row number to the global DOF index and create from it
// the global PETSc row index of this DOF.
int petscRowIdx = mapLocalGlobalDofs[*cursor] * nComponents + i;
int petscRowIdx =
meshDistributor->mapLocalToGlobal(*cursor) * nComponents + i;
if (isRankDof[*cursor]) {
if (meshDistributor->getIsRankDof(*cursor)) {
// === The current row DOF is a rank dof, so create the corresponding ===
// === nnz values directly on rank's nnz data. ===
// This is the local row index of the local PETSc matrix.
int localPetscRowIdx = petscRowIdx - rstart * nComponents;
int localPetscRowIdx =
petscRowIdx - meshDistributor->getRstart() * nComponents;
#if (DEBUG != 0)
if (localPetscRowIdx < 0 || localPetscRowIdx >= nRankRows) {
std::cout << "ERROR in rank: " << mpiRank << std::endl;
std::cout << "ERROR in rank: " << meshDistributor->getMpiRank() << std::endl;
std::cout << " Wrong r = " << localPetscRowIdx << " " << *cursor
<< " " << mapLocalGlobalDofs[*cursor] << " "
<< " " << meshDistributor->mapLocalToGlobal(*cursor) << " "
<< nRankRows << std::endl;
ERROR_EXIT("Should not happen!\n");
}
......@@ -262,19 +287,21 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (value(*icursor) != 0.0) {
int petscColIdx = mapLocalGlobalDofs[col(*icursor)] * nComponents + j;
int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
// The row DOF is a rank DOF, if also the column is a rank DOF,
// increment the d_nnz values for this row, otherwise the o_nnz value.
if (petscColIdx >= rstart * nComponents &&
petscColIdx < rstart * nComponents + nRankRows)
if (petscColIdx >= meshDistributor->getRstart() * nComponents &&
petscColIdx < meshDistributor->getRstart() * nComponents + nRankRows)
d_nnz[localPetscRowIdx]++;
else
o_nnz[localPetscRowIdx]++;
}
}
} else {
typedef std::map<int, DofContainer> RankToDofContainer;
// === The current row DOF is not a rank dof, i.e., it will be created ===
// === on this rank, but after this it will be send to another rank ===
// === matrix. So we need to send also the corresponding nnz structure ===
......@@ -282,17 +309,11 @@ namespace AMDiS {
// Find out who is the member of this DOF.
int sendToRank = -1;
for (RankToDofContainer::iterator it = recvDofs.begin();
it != recvDofs.end(); ++it) {
for (RankToDofContainer::iterator it = meshDistributor->getRecvDofs().begin();
it != meshDistributor->getRecvDofs().end(); ++it) {
for (DofContainer::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (**dofIt == *cursor) {
if (petscRowIdx == 6717) {
debug::writeDofMesh(mpiRank, *cursor, feSpace);
MSG("SEND DOF TO: %d/%d\n", it->first, *cursor);
}
sendToRank = it->first;
break;
}
......@@ -308,7 +329,8 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) {
if (value(*icursor) != 0.0) {
int petscColIdx = mapLocalGlobalDofs[col(*icursor)] * nComponents + j;
int petscColIdx =
meshDistributor->mapLocalToGlobal(col(*icursor)) * nComponents + j;
sendMatrixEntry[sendToRank].
push_back(std::make_pair(petscRowIdx, petscColIdx));
......@@ -323,9 +345,9 @@ namespace AMDiS {
// === Send and recv the nnz row structure to/from other ranks. ===
StdMpi<MatrixNnzEntry> stdMpi(mpiComm, true);
StdMpi<MatrixNnzEntry> stdMpi(meshDistributor->getMpiComm(), true);
stdMpi.send(sendMatrixEntry);
stdMpi.recv(sendDofs);
stdMpi.recv(meshDistributor->getSendDofs());
stdMpi.startCommunication<int>(MPI_INT);
// === Evaluate the nnz structure this rank got from other ranks and add it to ===
......@@ -338,13 +360,14 @@ namespace AMDiS {
int r = it->second[i].first;
int c = it->second[i].second;
int localRowIdx = r - rstart * nComponents;
int localRowIdx = r - meshDistributor->getRstart() * nComponents;
TEST_EXIT_DBG(localRowIdx >= 0 && localRowIdx < nRankRows)
("Got row index %d/%d (nRankRows = %d) from rank %d. Should not happen!\n",
r, localRowIdx, nRankRows, it->first);
if (c < rstart * nComponents || c >= rstart * nComponents + nRankRows)
if (c < meshDistributor->getRstart() * nComponents ||
c >= meshDistributor->getRstart() * nComponents + nRankRows)
o_nnz[localRowIdx]++;
else
d_nnz[localRowIdx]++;
......@@ -359,6 +382,8 @@ namespace AMDiS {
FUNCNAME("GlobalMatrixSolver::fillPetscMatrix()");
clock_t first = clock();
int nRankRows = meshDistributor->getNumberRankDofs() * nComponents;
int nOverallRows = meshDistributor->getNumberOverallDofs() * nComponents;
// === Create PETSc vector (rhs, solution and a temporary vector). ===
......@@ -374,14 +399,14 @@ namespace AMDiS {
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
if (!d_nnz || lastMeshChangeIndex != lastMeshNnz) {
if (!d_nnz || meshDistributor->getLastMeshChangeIndex() != lastMeshNnz) {
if (d_nnz) {
delete [] d_nnz;
delete [] o_nnz;
}
createPetscNnzStructure(mat);
lastMeshNnz = lastMeshChangeIndex;
lastMeshNnz = meshDistributor->getLastMeshChangeIndex();
}
// === Create PETSc matrix with the computed nnz data structure. ===
......@@ -394,8 +419,10 @@ namespace AMDiS {
#if (DEBUG != 0)
int a, b;
MatGetOwnershipRange(petscMatrix, &a, &b);
TEST_EXIT(a == rstart * nComponents)("Wrong matrix ownership range!\n");
TEST_EXIT(b == rstart * nComponents + nRankRows)("Wrong matrix ownership range!\n");
TEST_EXIT(a == meshDistributor->getRstart() * nComponents)
("Wrong matrix ownership range!\n");
TEST_EXIT(b == meshDistributor->getRstart() * nComponents + nRankRows)
("Wrong matrix ownership range!\n");
#endif
// === Transfer values from DOF matrices to the PETSc matrix. ===
......@@ -456,10 +483,11 @@ namespace AMDiS {
PetscScalar *vecPointer;
VecGetArray(petscSolVec, &vecPointer);
int nRankDofs = meshDistributor->getNumberRankDofs();
for (int i = 0; i < nComponents; i++) {
DOFVector<double> *dofvec = vec.getDOFVector(i);
for (int j = 0; j < nRankDofs; j++)
(*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i];
(*dofvec)[meshDistributor->mapLocalToGlobal(j)] = vecPointer[j * nComponents + i];
}
VecRestoreArray(petscSolVec, &vecPointer);
......@@ -467,7 +495,7 @@ namespace AMDiS {
// === Synchronize dofs at common dofs, i.e., dofs that correspond to more ===
// === than one partition. ===
synchVector(vec);
meshDistributor->synchVector(vec);
// === Print information about solution process. ===
......
AMDiS/src/parallel/GlobalMatrixSolver.h
View file @ c541effa
......@@ -33,12 +33,14 @@
#include "petscao.h"
namespace AMDiS {
class GlobalMatrixSolver : public ParallelDomainBase