From fb74c0af23ec3bbc5c03ff8998c2270eaaca1d92 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Wed, 15 Apr 2009 14:43:26 +0000
Subject: [PATCH] * On the way to parallel amdis

---
 AMDiS/libtool                      | 70 +++++++++++++++---------------
 AMDiS/src/DOFAdmin.h               | 20 +--------
 AMDiS/src/Mesh.cc                  | 21 +++++++++
 AMDiS/src/Mesh.h                   | 22 +++-------
 AMDiS/src/ParallelDomainProblem.cc | 18 +++++---
 AMDiS/src/ParallelDomainProblem.h  |  6 ++-
 6 files changed, 83 insertions(+), 74 deletions(-)

diff --git a/AMDiS/libtool b/AMDiS/libtool
index 3b667337..45ef3ca1 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -30,10 +30,10 @@
 # the same distribution terms that you use for the rest of that program.
 
 # A sed program that does not truncate output.
-SED="/bin/sed"
+SED="/usr/bin/sed"
 
 # Sed that helps us avoid accidentally triggering echo(1) options like -n.
-Xsed="/bin/sed -e 1s/^X//"
+Xsed="/usr/bin/sed -e 1s/^X//"
 
 # The HP-UX ksh and POSIX shell print the target directory to stdout
 # if CDPATH is set.
@@ -44,7 +44,7 @@ available_tags=" CXX F77"
 
 # ### BEGIN LIBTOOL CONFIG
 
-# Libtool was configured on host NWRW15:
+# Libtool was configured on host p2s154:
 
 # Shell to use when invoking shell scripts.
 SHELL="/bin/sh"
@@ -66,12 +66,12 @@ fast_install=yes
 
 # The host system.
 host_alias=
-host=i686-pc-linux-gnu
+host=x86_64-unknown-linux-gnu
 host_os=linux-gnu
 
 # The build system.
 build_alias=
-build=i686-pc-linux-gnu
+build=x86_64-unknown-linux-gnu
 build_os=linux-gnu
 
 # An echo program that does not interpret backslashes.
@@ -82,25 +82,25 @@ AR="ar"
 AR_FLAGS="cru"
 
 # A C compiler.
-LTCC="gcc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
 
 # LTCC compiler flags.
 LTCFLAGS="-g -O2"
 
 # A language-specific compiler.
-CC="gcc"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
 
 # Is the compiler the GNU C compiler?
 with_gcc=yes
 
-gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
+gcc_dir=`gcc -print-file-name=. | /usr/bin/sed 's,/\.$,,'`
 gcc_ver=`gcc -dumpversion`
 
 # An ERE matcher.
 EGREP="grep -E"
 
 # The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
 
 # Whether we need hard or soft links.
 LN_S="ln -s"
@@ -174,7 +174,7 @@ dlopen_self=unknown
 dlopen_self_static=unknown
 
 # Compiler flag to prevent dynamic linking.
-link_static_flag="-static"
+link_static_flag=""
 
 # Compiler flag to turn off builtin functions.
 no_builtin_flag=" -fno-builtin"
@@ -328,10 +328,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
 link_all_deplibs=unknown
 
 # Compile-time system search path for libraries
-sys_lib_search_path_spec=`echo " /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
+sys_lib_search_path_spec=`echo "/lib64 /usr/lib64 /usr/local/lib64" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
 
 # Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/octave-2.9.9 /usr/lib/qt-3.3/lib "
+sys_lib_dlsearch_path_spec="/lib64 /usr/lib64 /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
 
 # Fix the shell variable $srcfile for the compiler.
 fix_srcfile_path=""
@@ -6763,7 +6763,7 @@ build_old_libs=`case $build_libtool_libs in yes) $echo no;; *) $echo yes;; esac`
 # End:
 # ### BEGIN LIBTOOL TAG CONFIG: CXX
 
-# Libtool was configured on host NWRW15:
+# Libtool was configured on host p2s154:
 
 # Shell to use when invoking shell scripts.
 SHELL="/bin/sh"
@@ -6785,12 +6785,12 @@ fast_install=yes
 
 # The host system.
 host_alias=
-host=i686-pc-linux-gnu
+host=x86_64-unknown-linux-gnu
 host_os=linux-gnu
 
 # The build system.
 build_alias=
-build=i686-pc-linux-gnu
+build=x86_64-unknown-linux-gnu
 build_os=linux-gnu
 
 # An echo program that does not interpret backslashes.
@@ -6801,25 +6801,25 @@ AR="ar"
 AR_FLAGS="cru"
 
 # A C compiler.
-LTCC="gcc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
 
 # LTCC compiler flags.
 LTCFLAGS="-g -O2"
 
 # A language-specific compiler.
-CC="g++"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpiCC"
 
 # Is the compiler the GNU C compiler?
 with_gcc=yes
 
-gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
+gcc_dir=`gcc -print-file-name=. | /usr/bin/sed 's,/\.$,,'`
 gcc_ver=`gcc -dumpversion`
 
 # An ERE matcher.
 EGREP="grep -E"
 
 # The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
 
 # Whether we need hard or soft links.
 LN_S="ln -s"
@@ -6893,7 +6893,7 @@ dlopen_self=unknown
 dlopen_self_static=unknown
 
 # Compiler flag to prevent dynamic linking.
-link_static_flag="-static"
+link_static_flag=""
 
 # Compiler flag to turn off builtin functions.
 no_builtin_flag=" -fno-builtin"
@@ -6948,11 +6948,11 @@ striplib="strip --strip-unneeded"
 
 # Dependencies to place before the objects being linked to create a
 # shared library.
-predep_objects=`echo "/usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crti.o /usr/lib/gcc/i386-redhat-linux/4.1.2/crtbeginS.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
+predep_objects=`echo "/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
 
 # Dependencies to place after the objects being linked to create a
 # shared library.
-postdep_objects=`echo "/usr/lib/gcc/i386-redhat-linux/4.1.2/crtendS.o /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../crtn.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
+postdep_objects=`echo "/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
 
 # Dependencies to place before the objects being linked to create a
 # shared library.
@@ -6960,11 +6960,11 @@ predeps=""
 
 # Dependencies to place after the objects being linked to create a
 # shared library.
-postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
 
 # The library search path used internally by the compiler when linking
 # a shared library.
-compiler_lib_search_path=`echo "-L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
+compiler_lib_search_path=`echo "-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.." | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
 
 # Method to check whether dependent libraries are shared objects.
 deplibs_check_method="pass_all"
@@ -7044,10 +7044,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
 link_all_deplibs=unknown
 
 # Compile-time system search path for libraries
-sys_lib_search_path_spec=`echo " /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../i386-redhat-linux/4.1.2/ /usr/lib/gcc/i386-redhat-linux/4.1.2/../../../ /lib/i386-redhat-linux/4.1.2/ /lib/ /usr/lib/i386-redhat-linux/4.1.2/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
+sys_lib_search_path_spec=`echo "/lib64 /usr/lib64 /usr/local/lib64" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
 
 # Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/octave-2.9.9 /usr/lib/qt-3.3/lib "
+sys_lib_dlsearch_path_spec="/lib64 /usr/lib64 /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
 
 # Fix the shell variable $srcfile for the compiler.
 fix_srcfile_path=""
@@ -7071,7 +7071,7 @@ include_expsyms=""
 
 # ### BEGIN LIBTOOL TAG CONFIG: F77
 
-# Libtool was configured on host NWRW15:
+# Libtool was configured on host p2s154:
 
 # Shell to use when invoking shell scripts.
 SHELL="/bin/sh"
@@ -7093,12 +7093,12 @@ fast_install=yes
 
 # The host system.
 host_alias=
-host=i686-pc-linux-gnu
+host=x86_64-unknown-linux-gnu
 host_os=linux-gnu
 
 # The build system.
 build_alias=
-build=i686-pc-linux-gnu
+build=x86_64-unknown-linux-gnu
 build_os=linux-gnu
 
 # An echo program that does not interpret backslashes.
@@ -7109,7 +7109,7 @@ AR="ar"
 AR_FLAGS="cru"
 
 # A C compiler.
-LTCC="gcc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
 
 # LTCC compiler flags.
 LTCFLAGS="-g -O2"
@@ -7118,16 +7118,16 @@ LTCFLAGS="-g -O2"
 CC="g77"
 
 # Is the compiler the GNU C compiler?
-with_gcc=yes
+with_gcc=
 
-gcc_dir=`gcc -print-file-name=. | /bin/sed 's,/\.$,,'`
+gcc_dir=`gcc -print-file-name=. | /usr/bin/sed 's,/\.$,,'`
 gcc_ver=`gcc -dumpversion`
 
 # An ERE matcher.
 EGREP="grep -E"
 
 # The linker used to build libraries.
-LD="/usr/bin/ld"
+LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
 
 # Whether we need hard or soft links.
 LN_S="ln -s"
@@ -7355,10 +7355,10 @@ variables_saved_for_relink="PATH LD_LIBRARY_PATH LD_RUN_PATH GCC_EXEC_PREFIX COM
 link_all_deplibs=unknown
 
 # Compile-time system search path for libraries
-sys_lib_search_path_spec=`echo " /usr/lib/gcc/i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../../i386-redhat-linux/lib/i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../../i386-redhat-linux/lib/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../i386-redhat-linux/3.4.6/ /usr/lib/gcc/i386-redhat-linux/3.4.6/../../../ /lib/i386-redhat-linux/3.4.6/ /lib/ /usr/lib/i386-redhat-linux/3.4.6/ /usr/lib/" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
+sys_lib_search_path_spec=`echo "/lib64 /usr/lib64 /usr/local/lib64" | $SED -e "s@${gcc_dir}@\${gcc_dir}@g;s@${gcc_ver}@\${gcc_ver}@g"`
 
 # Run-time system search path for libraries
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/lib/octave-2.9.9 /usr/lib/qt-3.3/lib "
+sys_lib_dlsearch_path_spec="/lib64 /usr/lib64 /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
 
 # Fix the shell variable $srcfile for the compiler.
 fix_srcfile_path=""
diff --git a/AMDiS/src/DOFAdmin.h b/AMDiS/src/DOFAdmin.h
index c327d855..fb91a520 100644
--- a/AMDiS/src/DOFAdmin.h
+++ b/AMDiS/src/DOFAdmin.h
@@ -28,10 +28,6 @@
 #ifndef AMDIS_DOFADMIN_H
 #define AMDIS_DOFADMIN_H
 
-// ============================================================================
-// ===== includes =============================================================
-// ============================================================================
-
 #include "Global.h"
 #include "FixVec.h"
 #include "MemoryManager.h"
@@ -43,10 +39,6 @@
 
 namespace AMDiS {
 
-  // ============================================================================
-  // ===== forward declarations =================================================
-  // ============================================================================
-
   class Mesh;
   class FiniteElemSpace;
   class ElInfo;
@@ -57,10 +49,6 @@ namespace AMDiS {
 
   template<typename T> class DOFVector;
 
-  // ============================================================================
-  // ===== class DOFAdmin =======================================================
-  // ============================================================================
-
   /** \ingroup DOFAdministration
    * \brief
    * Holds all data about one set of DOFs. It includes information about used and
@@ -161,8 +149,6 @@ namespace AMDiS {
       return dofIndexedList.end();
     }
 
-    // ===== getting methods ======================================================
-
     /** \name getting methods
      * \{
      */
@@ -254,8 +240,6 @@ namespace AMDiS {
 
     /** \} */
 
-    // ===== setting methods ======================================================
-
     /** \name setting methods
      * \{
      */
@@ -305,10 +289,10 @@ namespace AMDiS {
      */
     void freeDOFIndex(int dof);
 
-    // ===== Serializable implementation =====
-  
+    ///
     void serialize(std::ostream &out);
 
+    ///
     void deserialize(std::istream &in);
 
   protected:
diff --git a/AMDiS/src/Mesh.cc b/AMDiS/src/Mesh.cc
index 026832e1..a0ad13f8 100644
--- a/AMDiS/src/Mesh.cc
+++ b/AMDiS/src/Mesh.cc
@@ -354,6 +354,27 @@ namespace AMDiS {
     nVertices = nRemainDofs;
   }
 
+  void Mesh::createContinuousDofOrdering(FiniteElemSpace *feSpace, unsigned int addC) 
+  {
+    FUNCNAME("Mesh::createContinuousDofOrdering()");
+
+    const BasisFunction* basisFcts = feSpace->getBasisFcts();
+
+    TEST_EXIT(dim == 2)("Not yet implemented!\n");
+    TEST_EXIT(basisFcts->getNumber() == 3)("Not yet implemented!\n");
+
+    std::set<DegreeOfFreedom*> dofs;
+    
+    TraverseStack stack;
+    ElInfo *elInfo = stack.traverseFirst(this, -1, Mesh::CALL_LEAF_EL);
+    while (elInfo) {
+      for (int i = 0; i < 3; i++) {
+	dofs.insert(const_cast<DegreeOfFreedom*>(elInfo->getElement()->getDOF(i)));
+      }
+      elInfo = stack.traverseNext(elInfo);
+    }
+  }
+
   int Mesh::traverse(int level, Flag flag, int (*el_fct)(ElInfo*))
   {
     FUNCNAME("Mesh::traverse()");
diff --git a/AMDiS/src/Mesh.h b/AMDiS/src/Mesh.h
index a7acddc4..4854b733 100644
--- a/AMDiS/src/Mesh.h
+++ b/AMDiS/src/Mesh.h
@@ -33,10 +33,6 @@
 #ifndef AMDIS_MESH_H
 #define AMDIS_MESH_H
 
-// ============================================================================
-// ===== includes =============================================================
-// ============================================================================
-
 #include "DOFAdmin.h"
 #include "Line.h"
 #include "Triangle.h"
@@ -53,10 +49,6 @@
 
 namespace AMDiS {
 
-  // ============================================================================
-  // ===== forward declarations =================================================
-  // ============================================================================
-
   template <typename T> class DimVec; 
   template <typename T> class VectorOfFixVecs;
   class Boundary;
@@ -73,10 +65,6 @@ namespace AMDiS {
   class DOFVectorDOF;
   class VertexVector;
 
-  // ============================================================================
-  // ===== class Mesh ===========================================================
-  // ============================================================================
-
   /** \ingroup Triangulation 
    * \brief
    * A Mesh holds all information about a triangulation. 
@@ -109,8 +97,6 @@ namespace AMDiS {
     static int newDOFFct2(ElInfo* e);
     /** \} */
 
-
-    // ==========================================================================
     /** \name getting methods
      * \{
      */
@@ -267,7 +253,6 @@ namespace AMDiS {
 
     /** \} */
 
-    // ==========================================================================
     /** \name setting methods
      * \{
      */
@@ -421,6 +406,13 @@ namespace AMDiS {
      */
     void removeMacroElements(std::vector<MacroElement*>& macros);
 
+    /* \brief
+     * Creates new numbers for all dofs such that the set of dof numbers is a
+     * continuous interval of natural numbers starting with 0. The constatnt addC
+     * may be added to the numbers.
+     */
+    void createContinuousDofOrdering(FiniteElemSpace *feSpace, unsigned int addC = 0);
+
     /// Frees the array of DOF pointers (see \ref createDOFPtrs)
     void freeDOFPtrs(DegreeOfFreedom **ptrs);
 
diff --git a/AMDiS/src/ParallelDomainProblem.cc b/AMDiS/src/ParallelDomainProblem.cc
index b793fadd..0aa12250 100644
--- a/AMDiS/src/ParallelDomainProblem.cc
+++ b/AMDiS/src/ParallelDomainProblem.cc
@@ -14,10 +14,11 @@ namespace AMDiS {
   ParallelDomainProblemBase::ParallelDomainProblemBase(const std::string& name,
 						       ProblemIterationInterface *iIF,
 						       ProblemTimeInterface *tIF,
-						       Mesh *m)
+						       FiniteElemSpace *fe)
     : iterationIF(iIF),
       timeIF(tIF),
-      mesh(m),
+      feSpace(fe),
+      mesh(fe->getMesh()),
       initialPartitionMesh(true),
       nRankDOFs(0)
   {
@@ -96,6 +97,8 @@ namespace AMDiS {
       }
     }
 
+    // === Remove all macro elements that are not part of the rank partition. ===
+
     std::vector<MacroElement*> macrosToRemove;
     for (std::deque<MacroElement*>::iterator it = mesh->firstMacroElement();
 	 it != mesh->endOfMacroElements();
@@ -110,6 +113,8 @@ namespace AMDiS {
 
     mesh->removeMacroElements(macrosToRemove);
 
+    // === Create local and global dofs ordering. ===
+
     int *gOrder = (int*)(malloc(sizeof(int) * rankDofs.size()));
     int *lOrder = (int*)(malloc(sizeof(int) * rankDofs.size()));
 
@@ -120,11 +125,14 @@ namespace AMDiS {
 
     int rstart = 0;
     MPI_Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
-
+    rstart -= nRankDOFs;
+    
     for (int i = 0; i < nRankDOFs; i++) {
-      lOrder[i] = rstart - nRankDOFs + i;
+      lOrder[i] = rstart + i;
     }
 
+    mesh->createContinuousDofOrdering(feSpace, rstart);
+
     AOCreateBasic(PETSC_COMM_WORLD, nRankDOFs, gOrder, lOrder, &applicationOrdering);
 
     free(gOrder);
@@ -187,7 +195,7 @@ namespace AMDiS {
   ParallelDomainProblemScal::ParallelDomainProblemScal(const std::string& name,
 						       ProblemScal *problem,
 						       ProblemInstatScal *problemInstat)
-    : ParallelDomainProblemBase(name, problem, problemInstat, problem->getMesh())
+    : ParallelDomainProblemBase(name, problem, problemInstat, problem->getFESpace())
   {
   }
 
diff --git a/AMDiS/src/ParallelDomainProblem.h b/AMDiS/src/ParallelDomainProblem.h
index 77995799..ac369330 100644
--- a/AMDiS/src/ParallelDomainProblem.h
+++ b/AMDiS/src/ParallelDomainProblem.h
@@ -27,6 +27,7 @@
 
 #include "ProblemTimeInterface.h"
 #include "ProblemIterationInterface.h"
+#include "FiniteElemSpace.h"
 #include "AdaptInfo.h"
 #include "petscao.h"
 #include "mpi.h"
@@ -45,7 +46,7 @@ namespace AMDiS {
     ParallelDomainProblemBase(const std::string& name,
 			      ProblemIterationInterface *iterationIF,
 			      ProblemTimeInterface *timeIF,
-			      Mesh *mesh);
+			      FiniteElemSpace *feSpace);
 
     virtual ~ParallelDomainProblemBase() {}
 
@@ -123,6 +124,9 @@ namespace AMDiS {
     /// Name of the problem (as used in the init files)
     std::string name;
 
+    /// Finite element space of the problem.
+    FiniteElemSpace *feSpace;
+
     /// Mesh of the problem.
     Mesh *mesh;
 
-- 
GitLab