Merge branch 'feature/openmp' into 'dev'

Feature/openmp

See merge request !23

AMDiS/CMakeLists.txt

@@ -16,12 +16,35 @@ endif ()
 # ------------------------------------------------------------------------------
 
 # some options to control the build process
-option(ENABLE_PARALLEL_DOMAIN "Use parallel domain decomposition"     false)
+option(ENABLE_MPI             "Compile with MPI compiler and library" false)
 option(ENABLE_UMFPACK         "Enable support for the UMFPACK solver" true)
 option(ENABLE_EXTENSIONS      "Use extensions for AMDiS"              false)
 option(ENABLE_HYPRE           "Use HYPRE AMG solver"                  false)
 option(ENABLE_PNG             "Use png reader/writer"                 false)
-option(ENABLE_SEQ_PETSC       "Use sequential PETSc solvers"          false)
+
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_PARALLEL_DOMAIN
+  "Use parallel domain decomposition" true
+  "ENABLE_MPI" false)
+
+# additional options for parallel_domain
+# {
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_ZOLTAN
+  "Add support for the Parallel Partitioning suite Zoltan" false
+  "ENABLE_PARALLEL_DOMAIN" false)
+
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_PARALLEL_SOLVERS
+  "Add some problem dependent solver, e.g. Feti, Navier-Stokes and Cahn-Hilliard" true
+  "ENABLE_PARALLEL_DOMAIN" false)
+
+include(CMakeDependentOption)
+cmake_dependent_option(ENABLE_BDDCML
+  "Add support for the BDDCML library" false
+  "ENABLE_PARALLEL_DOMAIN" false)
+# }
+
 
 # enable/disable some more features
 option(ENABLE_COMPRESSION "Use output compression for vtu and arh files" true)
@@ -33,6 +56,12 @@ mark_as_advanced(ENABLE_REINIT)
 option(ENABLE_COMPOSITE_FEM "Compile compositeFEM library" true)
 mark_as_advanced(ENABLE_COMPOSITE_FEM)
 
+option(ENABLE_OPENMP "Compile with OpenMP flags. (Experimental)" false)
+mark_as_advanced(ENABLE_OPENMP)
+
+option(ENABLE_SEQ_PETSC "Use sequential PETSc solvers" false)
+mark_as_advanced(ENABLE_SEQ_PETSC)
+
 option(BUILD_SHARED_LIBS "Build all libraries as shared or static, default: shared" true)
 mark_as_advanced(BUILD_SHARED_LIBS)
 

AMDiS/cmake/amdis_parallel.cmake

 if (ENABLE_PARALLEL_DOMAIN)
-    option(ENABLE_ZOLTAN "Add support for the Parallel Partitioning suite Zoltan" false)
-    option(ENABLE_PARALLEL_SOLVERS "Add some problem dependent solver, e.g. Feti, Navier-Stokes and Cahn-Hilliard" true)
-    option(ENABLE_BDDCML "Add support for the BDDCML library" false)
-    mark_as_advanced(ENABLE_PARALLEL_SOLVERS)
-    mark_as_advanced(ENABLE_BDDCML)
-
     set(PARALLEL_DOMAIN_AMDIS_SRC
         ${SOURCE_DIR}/parallel/DofComm.cc
         ${SOURCE_DIR}/parallel/CheckerPartitioner.cc

AMDiS/cmake/enable_mtl4.cmake

@@ -12,22 +12,14 @@ macro(enable_mtl4 _FLAGS_ _INC_DIRS_ _LIBS_)
     endif (LINK_EXECUTABLE)
     list(APPEND ${_FLAGS_} "-DMTL_ASSERT_FOR_THROW=1")
 
-
-    if (ENABLE_CXX11 OR AMDIS_NEED_CXX11)
-      set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
-      foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
-        list(APPEND ${_FLAGS_} "-DMTL_WITH_${feature}")
-      endforeach ()
-    endif ()
-
+    set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
+    foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
+      list(APPEND ${_FLAGS_} "-DMTL_WITH_${feature}")
+    endforeach ()
 
     if (ENABLE_OPENMP)
       find_package(OpenMP REQUIRED)
-      if (OPENMP_FOUND)
-        list(APPEND ${_FLAGS_} "-DMTL_WITH_OPENMP" ${OpenMP_CXX_FLAGS})
-      else ()
-        message(FATAL_ERROR "OpenMP not found")
-      endif (OPENMP_FOUND)
+      list(APPEND ${_FLAGS_} "-DMTL_WITH_OPENMP" "-DHAVE_OPENMP" ${OpenMP_CXX_FLAGS})
     endif (ENABLE_OPENMP)
 
 endmacro(enable_mtl4)

AMDiS/cmake3/AMDIS.cmake.in

-set(AMDIS_NEED_CXX11 @ENABLE_CXX11@)
 set(AMDIS_NEED_UMFPACK @ENABLE_UMFPACK@)
 set(AMDIS_NEED_COMPRESSION @ENABLE_COMPRESSION@)
 set(AMDIS_NEED_EXTENSIONS @ENABLE_EXTENSIONS@)
@@ -12,22 +11,23 @@ set(AMDIS_NEED_PNG @ENABLE_PNG@)
 set(AMDIS_NEED_SEQ_PETSC @ENABLE_SEQ_PETSC@)
 
 set(BUILD_SHARED_LIBS @BUILD_SHARED_LIBS@)
-set(CMAKE_BUILD_TYPE @CMAKE_BUILD_TYPE@)
 
 add_library(amdis_base INTERFACE)
 add_library(AMDiS ALIAS amdis_base)
 
 target_compile_definitions(amdis_base INTERFACE)
 
-if (AMDIS_NEED_CXX11)
-  target_enable_cxx11(AMDIS_NEED_CXX11 amdis_base INTERFACE)
-  if (NOT AMDIS_NEED_CXX11)
-    message(FATAL_ERROR "AMDiS was compiled with c++11 support, but the current compiler does not support this feature!")
-  endif (NOT AMDIS_NEED_CXX11)
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
-else ()
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=0)
-endif (AMDIS_NEED_CXX11)
+
+target_enable_cxx14(SUPPORTS_CXX14 amdis_base INTERFACE)
+if (NOT SUPPORTS_CXX14)
+  target_enable_cxx11(SUPPORTS_CXX11 amdis_base INTERFACE)
+endif ()
+
+if (NOT SUPPORTS_CXX11 AND NOT SUPPORTS_CXX14)
+  message(FATAL_ERROR "AMDiS was compiled with c++11 support, but the current compiler does not support this feature!")
+endif ()
+target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
+
 
 
 # try to detect the AMDiS include directory

AMDiS/cmake3/CMakeLists.txt

@@ -23,12 +23,19 @@ add_library(AMDiS::base ALIAS amdis_base)
 target_include_directories(amdis_base INTERFACE ${SOURCE_DIR})
 target_compile_definitions(amdis_base INTERFACE)
 
-target_enable_cxx11(ENABLE_CXX11 amdis_base INTERFACE)
-if (ENABLE_CXX11)
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
+
+target_enable_cxx14(SUPPORTS_CXX14 amdis_base INTERFACE)
+if (SUPPORTS_CXX14)
+  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX14=1)
 else ()
-  target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=0)
-endif (ENABLE_CXX11)
+  target_enable_cxx11(SUPPORTS_CXX11 amdis_base INTERFACE)
+endif ()
+
+if (NOT SUPPORTS_CXX11 AND NOT SUPPORTS_CXX14)
+  message(FATAL_ERROR "AMDiS needs c++11 support, but the current compiler does not support this feature!")
+endif ()
+target_compile_definitions(amdis_base INTERFACE AMDIS_HAS_CXX11=1)
+
 
 target_enable_mtl4(amdis_base INTERFACE)
 target_enable_boost(amdis_base INTERFACE)
@@ -65,6 +72,7 @@ add_library(amdis
     ${SOURCE_DIR}/DOFAdmin.cc
     ${SOURCE_DIR}/DOFIndexed.cc
     ${SOURCE_DIR}/DOFMatrix.cc
+    ${SOURCE_DIR}/DOFSerializer.cc
     ${SOURCE_DIR}/DOFVector.cc
     ${SOURCE_DIR}/DirichletBC.cc
     ${SOURCE_DIR}/DualTraverse.cc
@@ -171,6 +179,11 @@ add_library(amdis
     ${SOURCE_DIR}/ProblemStatDbg.cc
 )
 
+if (ENABLE_MPI)
+  target_enable_mpi(amdis_base INTERFACE)
+  target_compile_definitions(amdis_base INTERFACE HAVE_MPI=1)
+endif (ENABLE_MPI)
+
 include(amdis_parallel)     # adds sources to amdis
 include(amdis_extensions)   # -> target AMDiS::extensions
 include(muparser)           # -> target muparser

AMDiS/cmake3/amdis_parallel.cmake

 if (ENABLE_PARALLEL_DOMAIN)
-    option(ENABLE_ZOLTAN "Add support for the Parallel Partitioning suite Zoltan" false)
-    option(ENABLE_PARALLEL_SOLVERS "Add some problem dependent solver, e.g. Feti, Navier-Stokes and Cahn-Hilliard" true)
-    mark_as_advanced(ENABLE_PARALLEL_SOLVERS)
-
     add_library(amdis_parallel INTERFACE)
     target_sources(amdis PRIVATE
         ${SOURCE_DIR}/parallel/DofComm.cc

AMDiS/cmake3/muparser.cmake

@@ -17,10 +17,8 @@ add_library(muparser STATIC
 target_include_directories(muparser PUBLIC ${MUPARSER_INCLUDE_DIR})
 set_property(TARGET muparser PROPERTY POSITION_INDEPENDENT_CODE ON)
 
-if (ENABLE_CXX11)
-  target_enable_cxx11(MUPARSER_ENABLE_CXX11 muparser PRIVATE)
-  target_compile_definitions(muparser PRIVATE AMDIS_HAS_CXX11=1)
-endif (ENABLE_CXX11)
+target_enable_cxx11(MUPARSER_ENABLE_CXX11 muparser PRIVATE)
+target_compile_definitions(muparser PRIVATE AMDIS_HAS_CXX11=1)
 
 # specify how to install this target:
 # -----------------------------------

AMDiS/cmake3/target_enable_cxx11.cmake

 include(CheckCXXCompilerFlag)
 include(CheckCXXSourceCompiles)
 
+macro(target_enable_cxx14 RESULT_VAR _TARGET_ _SCOPE_)
+  check_cxx_compiler_flag("-std=c++14" COMPILER_SUPPORTS_CXX14_FLAG)
+
+  set(CXX14_CODE "
+  auto f() { return 1; }
+  int main(){
+    auto f1 = [](auto const& y) { return y; };
+    auto y0 = f();
+    auto y1 = f1(1);
+  }")
+
+  if (COMPILER_SUPPORTS_CXX14_FLAG)
+    set(CMAKE_REQUIRED_FLAGS "-std=c++14")
+    check_cxx_source_compiles("${CXX14_CODE}" CXX14_COMPILES_WITH_CXX14_FLAG)
+    set(CMAKE_REQUIRED_FLAGS "")
+  endif ()
+
+  if (COMPILER_SUPPORTS_CXX14_FLAG AND CXX14_COMPILES_WITH_CXX14_FLAG)
+    target_compile_options(${_TARGET_} ${_SCOPE_} "-std=c++14")
+    set(${RESULT_VAR} true CACHE BOOL "Enable C++14 compiler features" FORCE)
+  else ()
+    check_cxx_source_compiles("${CXX14_CODE}" CXX14_COMPILES)
+
+    if (CXX14_COMPILES)
+      set(${RESULT_VAR} true CACHE BOOL "Enable C++14 compiler features" FORCE)
+    else ()
+      set(${RESULT_VAR} false CACHE BOOL "Enable C++14 compiler features" FORCE)
+    endif ()
+  endif ()
+endmacro(target_enable_cxx14)
+
 macro(target_enable_cxx11 RESULT_VAR _TARGET_ _SCOPE_)
   check_cxx_compiler_flag("-std=c++11" COMPILER_SUPPORTS_CXX11_FLAG)
 

AMDiS/cmake3/target_enable_mtl4.cmake

 macro(target_enable_mtl4 _TARGET_ _SCOPE_)
-    if (${ARGC} GREATER 2)
-      set(LINK_EXECUTABLE ON)
-    else ()
-      set(LINK_EXECUTABLE OFF)
-    endif ()
+  if (${ARGC} GREATER 2)
+    set(LINK_EXECUTABLE ON)
+  else ()
+    set(LINK_EXECUTABLE OFF)
+  endif ()
 
-    if (LINK_EXECUTABLE)
-      target_include_directories(${_TARGET_} ${_SCOPE_} ${AMDIS_INCLUDE_DIR}/mtl4)
-    else ()
-      target_include_directories(${_TARGET_} ${_SCOPE_} ${BASE_DIR}/lib/mtl4)
-    endif (LINK_EXECUTABLE)
-    target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_ASSERT_FOR_THROW=1)
+  if (LINK_EXECUTABLE)
+    target_include_directories(${_TARGET_} ${_SCOPE_} ${AMDIS_INCLUDE_DIR}/mtl4)
+  else ()
+    target_include_directories(${_TARGET_} ${_SCOPE_} ${BASE_DIR}/lib/mtl4)
+  endif (LINK_EXECUTABLE)
+  target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_ASSERT_FOR_THROW=1)
 
-    if (ENABLE_CXX11)
-      set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
-      foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
-        target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_${feature})
-      endforeach ()
-    endif (ENABLE_CXX11)
-
-    if (ENABLE_OPENMP)
-        find_package(OpenMP REQUIRED)
-        if (OPENMP_FOUND)
-          target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_OPENMP)
-          target_compile_options(${_TARGET_} ${_SCOPE_} ${OpenMP_CXX_FLAGS})
-        else ()
-          message(FATAL_ERROR "OpenMP not found")
-        endif (OPENMP_FOUND)
-    endif (ENABLE_OPENMP)
+  set (CXX_ELEVEN_FEATURE_LIST "MOVE" "AUTO" "RANGEDFOR" "INITLIST" "STATICASSERT" "DEFAULTIMPL")
+  foreach (feature ${CXX_ELEVEN_FEATURE_LIST})
+    target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_${feature})
+  endforeach ()
 
+  if (ENABLE_OPENMP)
+    find_package(OpenMP REQUIRED)
+    target_compile_definitions(${_TARGET_} ${_SCOPE_} MTL_WITH_OPENMP HAVE_OPENMP)
+    target_compile_options(${_TARGET_} ${_SCOPE_} ${OpenMP_CXX_FLAGS})
+  endif (ENABLE_OPENMP)
 endmacro(target_enable_mtl4)

AMDiS/src/AMDiS.cc

@@ -115,7 +115,7 @@ namespace AMDiS {
       Parameters::init(initFileName);
     }
 
-#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#if defined(HAVE_PARALLEL_DOMAIN_AMDIS) || defined(HAVE_OPENMP) || defined(HAVE_MPI)
     Parameters::get("parallel->log main rank", Msg::outputMainRank);
 #endif
 

AMDiS/src/AdaptInfo.h

@@ -156,11 +156,8 @@ namespace AMDiS {
 	rosenbrockMode(false)
     {
       init();
-      char number[5];
-      for (int i = 0; i < size; i++) {
-	sprintf(number, "[%d]", i);
-	scalContents[i] = new ScalContent(name + std::string(number));
-      }
+      for (int i = 0; i < size; i++)
+	scalContents[i] = new ScalContent(name + "[" + std::to_string(i) + "]");
     }
 
     /// Destructor.

AMDiS/src/CouplingIterationInterface.cc

@@ -88,7 +88,7 @@ namespace AMDiS {
 
 
   /// Returns number of managed problems
-  int CouplingIterationInterface::getNumProblems()
+  int CouplingIterationInterface::getNumProblems() const
   {
     size_t num= 0;
     for (size_t i = 0; i < problems.size(); ++i)
@@ -122,7 +122,7 @@ namespace AMDiS {
 
 
   /// Returns the name of the problem.
-  std::string CouplingIterationInterface::getName(size_t number)
+  std::string CouplingIterationInterface::getName(size_t number) const
   {
     if (static_cast<size_t>(problems.size()) <= number)
       throw(std::runtime_error("Problem number out of range."));

AMDiS/src/CouplingIterationInterface.h

@@ -61,7 +61,7 @@ namespace AMDiS {
     /// Called after each adaption loop iteration.
     virtual void endIteration(AdaptInfo *adaptInfo);
 
-    virtual int getNumProblems();
+    virtual int getNumProblems() const;
 
     /// Returns number of managed problems
     virtual size_t getNumIterationInterfaces() { return problems.size(); }
@@ -75,8 +75,8 @@ namespace AMDiS {
     virtual ProblemIterationInterface *getIterationInterface(size_t number = 0) { return problems[number]; }
 
     /// Returns the name of the problem with the given number.
-    virtual std::string getName(size_t number);
-    virtual std::string getName() { return getName(0); }
+    virtual std::string getName(size_t number) const;
+    virtual std::string getName() const { return getName(0); }
 
     virtual void setSolveProblem(size_t number, bool flag = true) { solveProblem[number] = flag; }
     virtual void setSolveProblem(std::string name, bool flag = true);

AMDiS/src/CouplingProblemStat.h

@@ -75,8 +75,7 @@ namespace AMDiS {
       virtual void initialize(Flag initFlag,
 			      ProblemStatSeq *adoptProblem = NULL,
 			      Flag adoptFlag = INIT_NOTHING)
-      { FUNCNAME("CouplingProblemStat::initialize()");
-
+      {
 	super::initialize(initFlag - INIT_MESH);
 
 	const Flag DEFAULT_INIT = (INIT_FE_SPACE | INIT_MESH | CREATE_MESH | INIT_SYSTEM | INIT_SOLVER | INIT_ESTIMATOR | INIT_MARKER | INIT_FILEWRITER);
@@ -130,7 +129,7 @@ namespace AMDiS {
 
 	  problems[i]->componentMeshes.resize(nComponents + nAddComponents);
 
-	  for (size_t j = 0; j < nComponents + nAddComponents; j++) {
+	  for (int j = 0; j < nComponents + nAddComponents; j++) {
 	    // name of the mesh
 	    std::string meshName("");
 	    Parameters::get(problems[i]->getName() + "->mesh", meshName);
@@ -186,7 +185,7 @@ namespace AMDiS {
 	  problems[p]->componentSpaces.resize(nComponents + nAddComponents, NULL);
 	  problems[p]->traverseInfo.resize(nComponents);
 
-	  for (size_t i = 0; i < nComponents + nAddComponents; i++) {
+	  for (int i = 0; i < nComponents + nAddComponents; i++) {
 
 	    std::string componentString = "[" + to_string(i) + "]";
 
@@ -335,7 +334,7 @@ namespace AMDiS {
 
 
       /// Returns number of managed problems
-      virtual int getNumProblems()
+      virtual int getNumProblems() const
       {
 	return problems.size();
       }
@@ -348,17 +347,35 @@ namespace AMDiS {
       {
 	return problems[number];
       }
+      
+      virtual ProblemStatType const* getProblem(int number = 0) const
+      {
+	return problems[number];
+      }
 
       /// Returns \ref meshes[i]
-      inline Mesh* getMesh(int number = 0)
+      Mesh* getMesh(int number = 0)
+      {
+	return meshes[number];
+      }
+      
+      /// Returns \ref meshes[i]
+      Mesh const* getMesh(int number = 0) const
       {
 	return meshes[number];
       }
 
-      using super::getNumComponents;
       using super::getRefinementManager;
       using super::getCoarseningManager;
       using super::getName;
+      
+      virtual int getNumComponents() const override
+      {
+        int num = 0;
+        for (int i = 0; i < getNumProblems(); ++i)
+          num += getProblem(i)->getNumComponents();
+        return num;
+      }
 
     protected:
       /// unqiue mesh-dimension for all problems

AMDiS/src/DOFAdmin.cc

@@ -215,10 +215,15 @@ namespace AMDiS {
 
     TEST_EXIT(dofIndexed)("no dofIndexed\n");
 
-    if (dofIndexed->getSize() < size)
-      dofIndexed->resize(size);
-
-    dofIndexedList.push_back(dofIndexed);
+    #ifdef _OPENMP
+    #pragma omp critical (addDOFIndexed)
+    #endif
+    {
+      if (dofIndexed->getSize() < size)
+        dofIndexed->resize(size);
+
+      dofIndexedList.push_back(dofIndexed);
+    }
   }
 
 
@@ -226,18 +231,17 @@ namespace AMDiS {
   {
     FUNCNAME("DOFAdmin::removeDOFIndexed()");
 
-    bool removed = false;
-    std::list<DOFIndexedBase*>::iterator it;
-    std::list<DOFIndexedBase*>::iterator end = dofIndexedList.end();
-    for (it = dofIndexedList.begin(); it != end; ++it) {
-      if (*it == dofIndexed) {
-	dofIndexedList.erase(it);
-	removed = true;
-	break;
+    #ifdef _OPENMP
+    #pragma omp critical (removeDOFIndexed)
+    #endif
+    {
+      auto it = std::find(dofIndexedList.begin(), dofIndexedList.end(), dofIndexed);
+      if (it != dofIndexedList.end())
+        dofIndexedList.erase(it);
+      else {
+        ERROR_EXIT("DOFIndexed not in list!\n");
       }
     }
-
-    TEST_EXIT(removed)("DOFIndexed not in list!\n");
   }
 
 
@@ -255,16 +259,17 @@ namespace AMDiS {
   {
     FUNCNAME("DOFAdmin::removeDOFContainer()");
 
-    std::list<DOFContainer*>::iterator it;
-    std::list<DOFContainer*>::iterator end = dofContainerList.end();
-    for (it = dofContainerList.begin(); it != end; ++it) {
-      if (*it == cont) {
-	dofContainerList.erase(it);
-	return;
+    #ifdef _OPENMP
+    #pragma omp critical (removeDOFContainer)
+    #endif
+    {
+      auto it = std::find(dofContainerList.begin(), dofContainerList.end(), cont);
+      if (it != dofContainerList.end())
+        dofContainerList.erase(it);
+      else {
+        ERROR_EXIT("Container not in list!\n");
       }
     }
-
-    ERROR("Container not in list!\n");
   }
 
 

AMDiS/src/DOFAdmin.h

@@ -182,6 +182,12 @@ namespace AMDiS {
       return mesh;
     }
 
+    /// Returns \ref mesh
+    inline Mesh* getMesh()
+    {
+      return mesh;
+    }
+
     /// Returns \ref dofFree, the array denoting DOFs to be either free or used.
     inline const std::vector<bool>& getDofFree() const
     {

AMDiS/src/DOFMatrix.cc

@@ -43,7 +43,7 @@ namespace AMDiS {
   DOFMatrix::DOFMatrix()
     : rowFeSpace(NULL),
       colFeSpace(NULL),
-      elementMatrix(3, 3),
+      elementMatrix(0, 0),
       nRow(0),
       nCol(0),
       nnzPerRow(0),

AMDiS/src/DOFSerializer.cc

+#include "DOFSerializer.h"
+
+#include <algorithm>
+
+#include "DOFVector.h"
+#include "Global.h"
+#include "Traverse.h"
+
+namespace AMDiS
+{    
+    
+// satre values of DOFVector on all macro elements to internal container
+void DOFSerializer::gather(DOFVector<double> const* vec, std::vector<double>& values)
+{
+  std::fill(visited_.begin(), visited_.end(), false);
+  values.clear();
+    
+  for (auto* macroEl : mesh_->getMacroElements())
+    gather(macroEl->getIndex(), vec, values, false);
+}
+
+
+// satre values of DOFVector on macro element `macroIndex` to internal container
+void DOFSerializer::gather(int macroIndex, DOFVector<double> const* vec, std::vector<double>& values, bool reset)
+{
+  if (reset) {
+    std::fill(visited_.begin(), visited_.end(), false);
+    values.clear();
+  }
+  
+  TEST_EXIT(mesh_ == vec->getFeSpace()->getMesh())("Incompatible meshes!\n");
+  
+  TraverseStack stack;
+  ElInfo *elInfo = stack.traverseFirstOneMacro(mesh_, macroIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
+  while (elInfo) {
+    Element *el = elInfo->getElement();
+    
+    if (el->isLeaf()) {
+      gather(VERTEX, elInfo, vec, values);
+      if (mesh_->getDim() > 1)
+        gather(EDGE, elInfo, vec, values);
+      if (mesh_->getDim() == 3)
+        gather(FACE, elInfo, vec, values);
+      gather(CENTER, elInfo, vec, values);
+    }
+    
+    elInfo = stack.traverseNext(elInfo);
+  }
+}
+
+
+void DOFSerializer::gather(GeoIndex geo, ElInfo* elInfo, DOFVector<double> const* vec, std::vector<double>& values) 
+{
+  int nd;
+  if ((nd = admin_->getNumberOfDofs(geo)))  {
+    int entities = mesh_->getGeo(geo);
+    int nd0 = admin_->getNumberOfPreDofs(geo);
+    int n0 = mesh_->getNode(geo);
+    for (int n = 0; n < entities; n++) {
+      for(int d = 0; d < nd; d++) {
+        DegreeOfFreedom globalDof = elInfo->getElement()->getDof(n0 + n, nd0 + d);
+        TEST_EXIT(globalDof < visited_.size())("visited container not large enough!\n");
+        if (!visited_[globalDof]) {
+          visited_[globalDof] = true;
+          values.push_back((*vec)[globalDof]);
+        }
+      }
+    }
+  }
+}
+
+
+// assign stored values to DOFVector, for all macro elements
+void DOFSerializer::scatter(std::vector<double> const& values, DOFVector<double>* vec) const
+{
+  std::fill(visited_.begin(), visited_.end(), false);
+  counter_ = 0u;
+    
+  for (auto* macroEl : mesh_->getMacroElements())
+    scatter(macroEl->getIndex(), values, vec, false);
+}
+
+
+// assign stored values to DOFVector, on macroElement with index `macroIndex`