Merge branch 'feature/openmp' into 'dev'

Feature/openmp See merge request !23

Merge branch 'feature/openmp' into 'dev'
Feature/openmp See merge request !23
37ff109d · Praetorius, Simon · 31322270 · c7b8f81f · 37ff109d · 37ff109d
Commit 37ff109d authored Jan 11, 2018 by Praetorius, Simon
20 changed files
--- a/AMDiS/CMakeLists.txt
+++ b/AMDiS/CMakeLists.txt
@@ -16,12 +16,35 @@ endif ()
 # ------------------------------------------------------------------------------

 # some options to control the build process
-option(ENABLE_PARALLEL_DOMAIN "Use parallel domain decomposition"     false)
+option(ENABLE_MPI             "Compile with MPI compiler and library" false)
 option(ENABLE_UMFPACK         "Enable support for the UMFPACK solver" true)
 option(ENABLE_EXTENSIONS      "Use extensions for AMDiS"              false)
 option(ENABLE_HYPRE           "Use HYPRE AMG solver"                  false)
 option(ENABLE_PNG             "Use png reader/writer"                 false)
-option(ENABLE_SEQ_PETSC       "Use sequential PETSc solvers"          false)
+
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_PARALLEL_DOMAIN
+  "Use parallel domain decomposition" true
+  "ENABLE_MPI" false)
+
+# additional options for parallel_domain
+# {
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_ZOLTAN
+  "Add support for the Parallel Partitioning suite Zoltan" false
+  "ENABLE_PARALLEL_DOMAIN" false)
+
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_PARALLEL_SOLVERS
+  "Add some problem dependent solver, e.g. Feti, Navier-Stokes and Cahn-Hilliard" true
+  "ENABLE_PARALLEL_DOMAIN" false)
+
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_BDDCML
+  "Add support for the BDDCML library" false
+  "ENABLE_PARALLEL_DOMAIN" false)
+# }
+

 # enable/disable some more features
 option(ENABLE_COMPRESSION "Use output compression for vtu and arh files" true)
@@ -33,6 +56,12 @@ mark_as_advanced(ENABLE_REINIT)
 option(ENABLE_COMPOSITE_FEM "Compile compositeFEM library" true)
 mark_as_advanced(ENABLE_COMPOSITE_FEM)

+option(ENABLE_OPENMP "Compile with OpenMP flags. (Experimental)" false)
+mark_as_advanced(ENABLE_OPENMP)
+
+option(ENABLE_SEQ_PETSC "Use sequential PETSc solvers" false)
+mark_as_advanced(ENABLE_SEQ_PETSC)
+
 option(BUILD_SHARED_LIBS "Build all libraries as shared or static, default: shared" true)
 mark_as_advanced(BUILD_SHARED_LIBS)


--- a/AMDiS/cmake/amdis_parallel.cmake
+++ b/AMDiS/cmake/amdis_parallel.cmake
 if (ENABLE_PARALLEL_DOMAIN)
-    option(ENABLE_ZOLTAN "Add support for the Parallel Partitioning suite Zoltan" false)
-    option(ENABLE_PARALLEL_SOLVERS "Add some problem dependent solver, e.g. Feti, Navier-Stokes and Cahn-Hilliard" true)
-    option(ENABLE_BDDCML "Add support for the BDDCML library" false)
-    mark_as_advanced(ENABLE_PARALLEL_SOLVERS)
-    mark_as_advanced(ENABLE_BDDCML)
-
    set(PARALLEL_DOMAIN_AMDIS_SRC
        ${SOURCE_DIR}/parallel/DofComm.cc
        ${SOURCE_DIR}/parallel/CheckerPartitioner.cc

--- a/AMDiS/cmake/enable_mtl4.cmake
+++ b/AMDiS/cmake/enable_mtl4.cmake
@@ -12,22 +12,14 @@ macro(enable_mtl4 _FLAGS_ _INC_DIRS_ _LIBS_)
    endif (LINK_EXECUTABLE)
    list(APPEND ${_FLAGS_} "-DMTL_ASSERT_FOR_THROW=1")

-
-    if (ENABLE_CXX11 OR AMDIS_NEED_CXX11)
-      set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
-      foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
-        list(APPEND ${_FLAGS_} "-DMTL_WITH_${feature}")
-      endforeach ()
-    endif ()
-
+    set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
+    foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
+      list(APPEND ${_FLAGS_} "-DMTL_WITH_${feature}")
+    endforeach ()

    if (ENABLE_OPENMP)
      find_package(OpenMP REQUIRED)
-      if (OPENMP_FOUND)
-        list(APPEND ${_FLAGS_} "-DMTL_WITH_OPENMP" ${OpenMP_CXX_FLAGS})
-      else ()
-        message(FATAL_ERROR "OpenMP not found")
-      endif (OPENMP_FOUND)
+      list(APPEND ${_FLAGS_} "-DMTL_WITH_OPENMP" "-DHAVE_OPENMP" ${OpenMP_CXX_FLAGS})
    endif (ENABLE_OPENMP)

 endmacro(enable_mtl4)
--- a/AMDiS/cmake3/AMDIS.cmake.in
+++ b/AMDiS/cmake3/AMDIS.cmake.in
-set(AMDIS_NEED_CXX11 @ENABLE_CXX11@)
 set(AMDIS_NEED_UMFPACK @ENABLE_UMFPACK@)
 set(AMDIS_NEED_COMPRESSION @ENABLE_COMPRESSION@)
 set(AMDIS_NEED_EXTENSIONS @ENABLE_EXTENSIONS@)
@@ -12,22 +11,23 @@ set(AMDIS_NEED_PNG @ENABLE_PNG@)
 set(AMDIS_NEED_SEQ_PETSC @ENABLE_SEQ_PETSC@)

 set(BUILD_SHARED_LIBS @BUILD_SHARED_LIBS@)
-set(CMAKE_BUILD_TYPE @CMAKE_BUILD_TYPE@)

 add_library(amdis_base INTERFACE)
 add_library(AMDiS ALIAS amdis_base)

 target_compile_definitions(amdis_base INTERFACE)

-if (AMDIS_NEED_CXX11)
-  target_enable_cxx11(AMDIS_NEED_CXX11 amdis_base INTERFACE)
-  if (NOT AMDIS_NEED_CXX11)
-    message(FATAL_ERROR "AMDiS was compiled with c++11 support, but the current compiler does not support this feature!")
-  endif (NOT AMDIS_NEED_CXX11)
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
-else ()
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=0)
-endif (AMDIS_NEED_CXX11)
+
+target_enable_cxx14(SUPPORTS_CXX14 amdis_base INTERFACE)
+if (NOT SUPPORTS_CXX14)
+  target_enable_cxx11(SUPPORTS_CXX11 amdis_base INTERFACE)
+endif ()
+
+if (NOT SUPPORTS_CXX11 AND NOT SUPPORTS_CXX14)
+  message(FATAL_ERROR "AMDiS was compiled with c++11 support, but the current compiler does not support this feature!")
+endif ()
+target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
+


 # try to detect the AMDiS include directory

--- a/AMDiS/cmake3/CMakeLists.txt
+++ b/AMDiS/cmake3/CMakeLists.txt
@@ -23,12 +23,19 @@ add_library(AMDiS::base ALIAS amdis_base)
 target_include_directories(amdis_base INTERFACE ${SOURCE_DIR})
 target_compile_definitions(amdis_base INTERFACE)

-target_enable_cxx11(ENABLE_CXX11 amdis_base INTERFACE)
-if (ENABLE_CXX11)
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
+
+target_enable_cxx14(SUPPORTS_CXX14 amdis_base INTERFACE)
+if (SUPPORTS_CXX14)
+  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX14=1)
 else ()
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=0)
-endif (ENABLE_CXX11)
+  target_enable_cxx11(SUPPORTS_CXX11 amdis_base INTERFACE)
+endif ()
+
+if (NOT SUPPORTS_CXX11 AND NOT SUPPORTS_CXX14)
+  message(FATAL_ERROR "AMDiS needs c++11 support, but the current compiler does not support this feature!")
+endif ()
+target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
+

 target_enable_mtl4(amdis_base INTERFACE)
 target_enable_boost(amdis_base INTERFACE)
@@ -65,6 +72,7 @@ add_library(amdis
    ${SOURCE_DIR}/DOFAdmin.cc
    ${SOURCE_DIR}/DOFIndexed.cc
    ${SOURCE_DIR}/DOFMatrix.cc
+    ${SOURCE_DIR}/DOFSerializer.cc
    ${SOURCE_DIR}/DOFVector.cc
    ${SOURCE_DIR}/DirichletBC.cc
    ${SOURCE_DIR}/DualTraverse.cc
@@ -171,6 +179,11 @@ add_library(amdis
    ${SOURCE_DIR}/ProblemStatDbg.cc
 )

+if (ENABLE_MPI)
+  target_enable_mpi(amdis_base INTERFACE)
+  target_compile_definitions(amdis_base INTERFACE HAVE_MPI=1)
+endif (ENABLE_MPI)
+
 include(amdis_parallel)     # adds sources to amdis
 include(amdis_extensions)   # -> target AMDiS::extensions
 include(muparser)           # -> target muparser

--- a/AMDiS/cmake3/amdis_parallel.cmake
+++ b/AMDiS/cmake3/amdis_parallel.cmake
 if (ENABLE_PARALLEL_DOMAIN)
-    option(ENABLE_ZOLTAN "Add support for the Parallel Partitioning suite Zoltan" false)
-    option(ENABLE_PARALLEL_SOLVERS "Add some problem dependent solver, e.g. Feti, Navier-Stokes and Cahn-Hilliard" true)
-    mark_as_advanced(ENABLE_PARALLEL_SOLVERS)
-
    add_library(amdis_parallel INTERFACE)
    target_sources(amdis PRIVATE
        ${SOURCE_DIR}/parallel/DofComm.cc

--- a/AMDiS/cmake3/muparser.cmake
+++ b/AMDiS/cmake3/muparser.cmake
@@ -17,10 +17,8 @@ add_library(muparser STATIC
 target_include_directories(muparser PUBLIC ${MUPARSER_INCLUDE_DIR})
 set_property(TARGET muparser PROPERTY POSITION_INDEPENDENT_CODE ON)

-if (ENABLE_CXX11)
-  target_enable_cxx11(MUPARSER_ENABLE_CXX11 muparser PRIVATE)
-  target_compile_definitions(muparser PRIVATE AMDIS_HAS_CXX11=1)
-endif (ENABLE_CXX11)
+target_enable_cxx11(MUPARSER_ENABLE_CXX11 muparser PRIVATE)
+target_compile_definitions(muparser PRIVATE AMDIS_HAS_CXX11=1)

 # specify how to install this target:
 # -----------------------------------

--- a/AMDiS/cmake3/target_enable_cxx11.cmake
+++ b/AMDiS/cmake3/target_enable_cxx11.cmake
 include(CheckCXXCompilerFlag)
 include(CheckCXXSourceCompiles)

+macro(target_enable_cxx14 RESULT_VAR _TARGET_ _SCOPE_)
+  check_cxx_compiler_flag("-std=c++14" COMPILER_SUPPORTS_CXX14_FLAG)
+
+  set(CXX14_CODE "
+  auto f() { return 1; }
+  int main(){
+    auto f1 = [](auto const& y) { return y; };
+    auto y0 = f();
+    auto y1 = f1(1);
+  }")
+
+  if (COMPILER_SUPPORTS_CXX14_FLAG)
+    set(CMAKE_REQUIRED_FLAGS "-std=c++14")
+    check_cxx_source_compiles("${CXX14_CODE}" CXX14_COMPILES_WITH_CXX14_FLAG)
+    set(CMAKE_REQUIRED_FLAGS "")
+  endif ()
+
+  if (COMPILER_SUPPORTS_CXX14_FLAG AND CXX14_COMPILES_WITH_CXX14_FLAG)
+    target_compile_options(${_TARGET_} ${_SCOPE_} "-std=c++14")
+    set(${RESULT_VAR} true CACHE BOOL "Enable C++14 compiler features" FORCE)
+  else ()
+    check_cxx_source_compiles("${CXX14_CODE}" CXX14_COMPILES)
+
+    if (CXX14_COMPILES)
+      set(${RESULT_VAR} true CACHE BOOL "Enable C++14 compiler features" FORCE)
+    else ()
+      set(${RESULT_VAR} false CACHE BOOL "Enable C++14 compiler features" FORCE)
+    endif ()
+  endif ()
+endmacro(target_enable_cxx14)
+
 macro(target_enable_cxx11 RESULT_VAR _TARGET_ _SCOPE_)
  check_cxx_compiler_flag("-std=c++11" COMPILER_SUPPORTS_CXX11_FLAG)


--- a/AMDiS/cmake3/target_enable_mtl4.cmake
+++ b/AMDiS/cmake3/target_enable_mtl4.cmake
 macro(target_enable_mtl4 _TARGET_ _SCOPE_)
-    if (${ARGC} GREATER 2)
-      set(LINK_EXECUTABLE ON)
-    else ()
-      set(LINK_EXECUTABLE OFF)
-    endif ()
+  if (${ARGC} GREATER 2)
+    set(LINK_EXECUTABLE ON)
+  else ()
+    set(LINK_EXECUTABLE OFF)
+  endif ()

-    if (LINK_EXECUTABLE)
-      target_include_directories(${_TARGET_} ${_SCOPE_} ${AMDIS_INCLUDE_DIR}/mtl4)
-    else ()
-      target_include_directories(${_TARGET_} ${_SCOPE_} ${BASE_DIR}/lib/mtl4)
-    endif (LINK_EXECUTABLE)
-    target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_ASSERT_FOR_THROW=1)
+  if (LINK_EXECUTABLE)
+    target_include_directories(${_TARGET_} ${_SCOPE_} ${AMDIS_INCLUDE_DIR}/mtl4)
+  else ()
+    target_include_directories(${_TARGET_} ${_SCOPE_} ${BASE_DIR}/lib/mtl4)
+  endif (LINK_EXECUTABLE)
+  target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_ASSERT_FOR_THROW=1)

-    if (ENABLE_CXX11)
-      set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
-      foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
-        target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_${feature})
-      endforeach ()
-    endif (ENABLE_CXX11)
-
-    if (ENABLE_OPENMP)
-        find_package(OpenMP REQUIRED)
-        if (OPENMP_FOUND)
-          target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_OPENMP)
-          target_compile_options(${_TARGET_} ${_SCOPE_} ${OpenMP_CXX_FLAGS})
-        else ()
-          message(FATAL_ERROR "OpenMP not found")
-        endif (OPENMP_FOUND)
-    endif (ENABLE_OPENMP)
+  set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
+  foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
+    target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_${feature})
+  endforeach ()

+  if (ENABLE_OPENMP)
+    find_package(OpenMP REQUIRED)
+    target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_OPENMP HAVE_OPENMP)
+    target_compile_options(${_TARGET_} ${_SCOPE_} ${OpenMP_CXX_FLAGS})
+  endif (ENABLE_OPENMP)
 endmacro(target_enable_mtl4)
--- a/AMDiS/src/AMDiS.cc
+++ b/AMDiS/src/AMDiS.cc
@@ -115,7 +115,7 @@ namespace AMDiS {
      Parameters::init(initFileName);
    }

-#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#if defined(HAVE_PARALLEL_DOMAIN_AMDIS) || defined(HAVE_OPENMP) || defined(HAVE_MPI)
    Parameters::get("parallel->log main rank", Msg::outputMainRank);
 #endif


--- a/AMDiS/src/AdaptInfo.h
+++ b/AMDiS/src/AdaptInfo.h
@@ -156,11 +156,8 @@ namespace AMDiS {
 	rosenbrockMode(false)
    {
      init();
-      char number[5];
-      for (int i = 0; i < size; i++) {
-	sprintf(number, "[%d]", i);
-	scalContents[i] = new ScalContent(name + std::string(number));
-      }
+      for (int i = 0; i < size; i++)
+	scalContents[i] = new ScalContent(name + "[" + std::to_string(i) + "]");
    }

    /// Destructor.

--- a/AMDiS/src/CouplingIterationInterface.cc
+++ b/AMDiS/src/CouplingIterationInterface.cc
@@ -88,7 +88,7 @@ namespace AMDiS {


  /// Returns number of managed problems
-  int CouplingIterationInterface::getNumProblems()
+  int CouplingIterationInterface::getNumProblems() const
  {
    size_t num= 0;
    for (size_t i = 0; i < problems.size(); ++i)
@@ -122,7 +122,7 @@ namespace AMDiS {


  /// Returns the name of the problem.
-  std::string CouplingIterationInterface::getName(size_t number)
+  std::string CouplingIterationInterface::getName(size_t number) const
  {
    if (static_cast<size_t>(problems.size()) <= number)
      throw(std::runtime_error("Problem number out of range."));

--- a/AMDiS/src/CouplingIterationInterface.h
+++ b/AMDiS/src/CouplingIterationInterface.h
@@ -61,7 +61,7 @@ namespace AMDiS {
    /// Called after each adaption loop iteration.
    virtual void endIteration(AdaptInfo *adaptInfo);

-    virtual int getNumProblems();
+    virtual int getNumProblems() const;

    /// Returns number of managed problems
    virtual size_t getNumIterationInterfaces() { return problems.size(); }
@@ -75,8 +75,8 @@ namespace AMDiS {
    virtual ProblemIterationInterface *getIterationInterface(size_t number = 0) { return problems[number]; }

    /// Returns the name of the problem with the given number.
-    virtual std::string getName(size_t number);
-    virtual std::string getName() { return getName(0); }
+    virtual std::string getName(size_t number) const;
+    virtual std::string getName() const { return getName(0); }

    virtual void setSolveProblem(size_t number, bool flag = true) { solveProblem[number] = flag; }
    virtual void setSolveProblem(std::string name, bool flag = true);

--- a/AMDiS/src/CouplingProblemStat.h
+++ b/AMDiS/src/CouplingProblemStat.h
@@ -75,8 +75,7 @@ namespace AMDiS {
      virtual void initialize(Flag initFlag,
 			      ProblemStatSeq *adoptProblem = NULL,
 			      Flag adoptFlag = INIT_NOTHING)
-      { FUNCNAME("CouplingProblemStat::initialize()");
-
+      {
 	super::initialize(initFlag - INIT_MESH);

 	const Flag DEFAULT_INIT = (INIT_FE_SPACE | INIT_MESH | CREATE_MESH | INIT_SYSTEM | INIT_SOLVER | INIT_ESTIMATOR | INIT_MARKER | INIT_FILEWRITER);
@@ -130,7 +129,7 @@ namespace AMDiS {

 	  problems[i]->componentMeshes.resize(nComponents + nAddComponents);

-	  for (size_t j = 0; j < nComponents + nAddComponents; j++) {
+	  for (int j = 0; j < nComponents + nAddComponents; j++) {
 	    // name of the mesh
 	    std::string meshName("");
 	    Parameters::get(problems[i]->getName() + "->mesh", meshName);
@@ -186,7 +185,7 @@ namespace AMDiS {
 	  problems[p]->componentSpaces.resize(nComponents + nAddComponents, NULL);
 	  problems[p]->traverseInfo.resize(nComponents);

-	  for (size_t i = 0; i < nComponents + nAddComponents; i++) {
+	  for (int i = 0; i < nComponents + nAddComponents; i++) {

 	    std::string componentString = "[" + to_string(i) + "]";

@@ -335,7 +334,7 @@ namespace AMDiS {


      /// Returns number of managed problems
-      virtual int getNumProblems()
+      virtual int getNumProblems() const
      {
 	return problems.size();
      }
@@ -348,17 +347,35 @@ namespace AMDiS {
      {
 	return problems[number];
      }
+      
+      virtual ProblemStatType const* getProblem(int number = 0) const
+      {
+	return problems[number];
+      }

      /// Returns \ref meshes[i]
-      inline Mesh* getMesh(int number = 0)
+      Mesh* getMesh(int number = 0)
+      {
+	return meshes[number];
+      }
+      
+      /// Returns \ref meshes[i]
+      Mesh const* getMesh(int number = 0) const
      {
 	return meshes[number];
      }

-      using super::getNumComponents;
      using super::getRefinementManager;
      using super::getCoarseningManager;
      using super::getName;
+      
+      virtual int getNumComponents() const override
+      {
+        int num = 0;
+        for (int i = 0; i < getNumProblems(); ++i)
+          num += getProblem(i)->getNumComponents();
+        return num;
+      }

    protected:
      /// unqiue mesh-dimension for all problems

--- a/AMDiS/src/DOFAdmin.cc
+++ b/AMDiS/src/DOFAdmin.cc
@@ -215,10 +215,15 @@ namespace AMDiS {

    TEST_EXIT(dofIndexed)("no dofIndexed\n");

-    if (dofIndexed->getSize() < size)
-      dofIndexed->resize(size);
-
-    dofIndexedList.push_back(dofIndexed);
+    #ifdef _OPENMP
+    #pragma omp critical (addDOFIndexed)
+    #endif
+    {
+      if (dofIndexed->getSize() < size)
+        dofIndexed->resize(size);
+
+      dofIndexedList.push_back(dofIndexed);
+    }
  }


@@ -226,18 +231,17 @@ namespace AMDiS {
  {
    FUNCNAME("DOFAdmin::removeDOFIndexed()");

-    bool removed = false;
-    std::list<DOFIndexedBase*>::iterator it;
-    std::list<DOFIndexedBase*>::iterator end = dofIndexedList.end();
-    for (it = dofIndexedList.begin(); it != end; ++it) {
-      if (*it == dofIndexed) {
-	dofIndexedList.erase(it);
-	removed = true;
-	break;
+    #ifdef _OPENMP
+    #pragma omp critical (removeDOFIndexed)
+    #endif
+    {
+      auto it = std::find(dofIndexedList.begin(), dofIndexedList.end(), dofIndexed);
+      if (it != dofIndexedList.end())
+        dofIndexedList.erase(it);
+      else {
+        ERROR_EXIT("DOFIndexed not in list!\n");
      }
    }
-
-    TEST_EXIT(removed)("DOFIndexed not in list!\n");
  }


@@ -255,16 +259,17 @@ namespace AMDiS {
  {
    FUNCNAME("DOFAdmin::removeDOFContainer()");

-    std::list<DOFContainer*>::iterator it;
-    std::list<DOFContainer*>::iterator end = dofContainerList.end();
-    for (it = dofContainerList.begin(); it != end; ++it) {
-      if (*it == cont) {
-	dofContainerList.erase(it);
-	return;
+    #ifdef _OPENMP
+    #pragma omp critical (removeDOFContainer)
+    #endif
+    {
+      auto it = std::find(dofContainerList.begin(), dofContainerList.end(), cont);
+      if (it != dofContainerList.end())
+        dofContainerList.erase(it);
+      else {
+        ERROR_EXIT("Container not in list!\n");
      }
    }
-
-    ERROR("Container not in list!\n");
  }



--- a/AMDiS/src/DOFAdmin.h
+++ b/AMDiS/src/DOFAdmin.h
@@ -182,6 +182,12 @@ namespace AMDiS {
      return mesh;
    }

+    /// Returns \ref mesh
+    inline Mesh* getMesh()
+    {
+      return mesh;
+    }
+
    /// Returns \ref dofFree, the array denoting DOFs to be either free or used.
    inline const std::vector<bool>& getDofFree() const
    {

--- a/AMDiS/src/DOFMatrix.cc
+++ b/AMDiS/src/DOFMatrix.cc
@@ -43,7 +43,7 @@ namespace AMDiS {
  DOFMatrix::DOFMatrix()
    : rowFeSpace(NULL),
      colFeSpace(NULL),
-      elementMatrix(3, 3),
+      elementMatrix(0, 0),
      nRow(0),
      nCol(0),
      nnzPerRow(0),

--- a/AMDiS/src/DOFSerializer.cc
+++ b/AMDiS/src/DOFSerializer.cc
+#include "DOFSerializer.h"
+
+#include <algorithm>
+
+#include "DOFVector.h"
+#include "Global.h"
+#include "Traverse.h"
+
+namespace AMDiS
+{    
+    
+// satre values of DOFVector on all macro elements to internal container
+void DOFSerializer::gather(DOFVector<double> const* vec, std::vector<double>& values)
+{
+  std::fill(visited_.begin(), visited_.end(), false);
+  values.clear();
+    
+  for (auto* macroEl : mesh_->getMacroElements())
+    gather(macroEl->getIndex(), vec, values, false);
+}
+
+
+// satre values of DOFVector on macro element `macroIndex` to internal container
+void DOFSerializer::gather(int macroIndex, DOFVector<double> const* vec, std::vector<double>& values, bool reset)
+{
+  if (reset) {
+    std::fill(visited_.begin(), visited_.end(), false);
+    values.clear();
+  }
+  
+  TEST_EXIT(mesh_ == vec->getFeSpace()->getMesh())("Incompatible meshes!\n");
+  
+  TraverseStack stack;
+  ElInfo *elInfo = stack.traverseFirstOneMacro(mesh_, macroIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
+  while (elInfo) {
+    Element *el = elInfo->getElement();
+    
+    if (el->isLeaf()) {
+      gather(VERTEX, elInfo, vec, values);
+      if (mesh_->getDim() > 1)
+        gather(EDGE, elInfo, vec, values);
+      if (mesh_->getDim() == 3)
+        gather(FACE, elInfo, vec, values);
+      gather(CENTER, elInfo, vec, values);
+    }
+    
+    elInfo = stack.traverseNext(elInfo);
+  }
+}
+
+
+void DOFSerializer::gather(GeoIndex geo, ElInfo* elInfo, DOFVector<double> const* vec, std::vector<double>& values) 
+{
+  int nd;
+  if ((nd = admin_->getNumberOfDofs(geo)))  {
+    int entities = mesh_->getGeo(geo);
+    int nd0 = admin_->getNumberOfPreDofs(geo);
+    int n0 = mesh_->getNode(geo);
+    for (int n = 0; n < entities; n++) {
+      for(int d = 0; d < nd; d++) {
+        DegreeOfFreedom globalDof = elInfo->getElement()->getDof(n0 + n, nd0 + d);
+        TEST_EXIT(globalDof < visited_.size())("visited container not large enough!\n");
+        if (!visited_[globalDof]) {
+          visited_[globalDof] = true;
+          values.push_back((*vec)[globalDof]);
+        }
+      }
+    }
+  }
+}
+
+
+// assign stored values to DOFVector, for all macro elements
+void DOFSerializer::scatter(std::vector<double> const& values, DOFVector<double>* vec) const
+{
+  std::fill(visited_.begin(), visited_.end(), false);
+  counter_ = 0u;
+    
+  for (auto* macroEl : mesh_->getMacroElements())
+    scatter(macroEl->getIndex(), values, vec, false);
+}
+
+
+// assign stored values to DOFVector, on macroElement with index `macroIndex`