Work on parallelization, merge to deimos.

AMDiS/libtool

@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
 LN_S="ln -s"
 
 # What is the maximum length of a command?
-max_cmd_len=98304
+max_cmd_len=1572864
 
 # Object file suffix (normally "o").
 objext=o
@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
 old_postuninstall_cmds=""
 
 # A C compiler.
-LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
+LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
 
 # LTCC compiler flags.
 LTCFLAGS="-g -O2"
@@ -233,10 +233,10 @@ finish_eval=""
 hardcode_into_libs=yes
 
 # Compile-time system search path for libraries.
-sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
+sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
 
 # Run-time system search path for libraries.
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
 
 # Whether dlopen is supported.
 dlopen_support=unknown
@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
 old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
 
 # A language specific compiler.
-CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
+CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
 
 # Is the compiler the GNU compiler?
 with_gcc=yes
@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
 old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
 
 # A language specific compiler.
-CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
+CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
 
 # Is the compiler the GNU compiler?
 with_gcc=yes
@@ -9039,17 +9039,17 @@ file_list_spec=""
 hardcode_action=immediate
 
 # The directories searched by this compiler when creating a shared library.
-compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
+compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
 
 # Dependencies to place before and after the objects being linked to
 # create a shared library.
-predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
-postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
+predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
+postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
 predeps=""
-postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
 
 # The library search path used internally by the compiler when linking
 # a shared library.
-compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
+compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
 
 # ### END LIBTOOL TAG CONFIG: CXX

AMDiS/src/MeshStructure.cc

@@ -246,6 +246,8 @@ namespace AMDiS {
     else
       elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
 
+    MSG("FIT 1\n");
+
     while (elInfo) {
       Element *element = elInfo->getElement();
 
@@ -281,8 +283,10 @@ namespace AMDiS {
 
     // refine mesh
     bool finished = true;
-
+    MSG("FIT 2\n");
     do {
+      MSG("RUN ON EL %d\n", macroElIndex);
+
       finished = true;
       if (macroElIndex == -1)
 	elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
@@ -303,18 +307,24 @@ namespace AMDiS {
 #if (DEBUG != 0)
 	int oldMeshIndex = mesh->getChangeIndex();
 #endif
-	
+
+	MSG("AND REFINE!\n");
+
 	if (macroElIndex == -1)
 	  manager->refineMesh(mesh);
 	else
 	  manager->refineMacroElement(mesh, macroElIndex);
-	
+
+	MSG("AND BACK!\n");
+
 #if (DEBUG != 0)
 	TEST_EXIT(oldMeshIndex != mesh->getChangeIndex())
 	  ("Mesh has not been changed by refinement procedure!\n");
 #endif
       }
     } while (!finished);
+
+    MSG("FIT 3\n");
   }
 
 
@@ -366,19 +376,21 @@ namespace AMDiS {
   {
     FUNCNAME("MeshStructure::getMeshStructureValues()");
 
+    TEST_EXIT_DBG(mesh)("No mesh defined!\n");
+    TEST_EXIT_DBG(vec)("No DOFVector defined!\n");
+
     values.clear();
 
     TraverseStack stack;
     ElInfo *elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1, 
 						 Mesh::CALL_EVERY_EL_PREORDER);
     while (elInfo) {
-      if (elInfo->getLevel() == 0) {
+      if (elInfo->getLevel() == 0)
 	for (int i = 0; i < mesh->getGeo(VERTEX); i++)
 	  values.push_back((*vec)[elInfo->getElement()->getDof(i, 0)]);
-      } else {
-	if (!elInfo->getElement()->isLeaf())
-	  values.push_back((*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)]);
-      }
+
+      if (!elInfo->getElement()->isLeaf())
+	values.push_back((*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)]);      
 
       elInfo = stack.traverseNext(elInfo);
     }
@@ -392,7 +404,10 @@ namespace AMDiS {
   {
     FUNCNAME("MeshStructure::setMeshStructureValues()");
 
-    TEST_EXIT_DBG(values.size() >= mesh->getGeo(VERTEX))("Should not happen!\n");
+    TEST_EXIT_DBG(mesh)("No mesh defined!\n");
+    TEST_EXIT_DBG(vec)("No DOFVector defined!\n");
+    TEST_EXIT_DBG(static_cast<int>(values.size()) >= mesh->getGeo(VERTEX))
+      ("Should not happen!\n");
 
     unsigned int counter = 0;
 
@@ -400,16 +415,15 @@ namespace AMDiS {
     ElInfo *elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1, 
 						 Mesh::CALL_EVERY_EL_PREORDER);
     while (elInfo) {
-      if (elInfo->getLevel() == 0) {
+      if (elInfo->getLevel() == 0)
 	for (int i = 0; i < mesh->getGeo(VERTEX); i++)
-	  (*vec)[elInfo->getElement()->getDof(i, 0)] = values[counter++];
-      } else {
-	if (!elInfo->getElement()->isLeaf()) {
-	  TEST_EXIT_DBG(counter < values.size())("Should not happen!\n");
+	  (*vec)[elInfo->getElement()->getDof(i, 0)] = values[counter++];      
 
-	  (*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)] =
-	    values[counter++];
-	}
+      if (!elInfo->getElement()->isLeaf()) {
+	TEST_EXIT_DBG(counter < values.size())("Should not happen!\n");
+	
+	(*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)] =
+	  values[counter++];      
       }
 
       elInfo = stack.traverseNext(elInfo);

AMDiS/src/RefinementManager.cc

@@ -90,6 +90,9 @@ namespace AMDiS {
     
     while (doMoreRecursiveRefine) {
       doMoreRecursiveRefine = false;
+
+      MSG("HERE 1\n");
+
       ElInfo *elInfo = 
 	stack->traverseFirstOneMacro(mesh, macroElIndex, -1, 
 				     Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
@@ -100,20 +103,28 @@ namespace AMDiS {
 	    doMoreRecursiveRefine || (elInfo->getElement()->getMark() > 1);
 	  elInfo = refineFunction(elInfo);
 	}	
+	MSG("WIRKLICH HERE? 1\n");
 	elInfo = stack->traverseNext(elInfo);
+	MSG("WIRKLICH HERE? 2\n");
       }
+
+      MSG("HERE 2\n");
     }
 
 
+    MSG("HERE 3\n");
+
     if (newCoords)
-      setNewCoords(); // call of sub-class method
+      setNewCoords(macroElIndex); // call of sub-class method
 
+    MSG("HERE 4\n");
     delete stack;
-
+        MSG("HERE 5\n");
     nElements -= mesh->getNumberOfLeaves();
-   
+       MSG("HERE 6\n");
     if (nElements != 0)
       aMesh->incChangeIndex();    
+    MSG("HERE 7\n");
   }
 
 }

AMDiS/src/RefinementManager.h

@@ -51,7 +51,7 @@ namespace AMDiS {
      * Generates new coordinates on curved boundaries. Can be overriden by
      * sub classes if used.
      */
-    virtual void setNewCoords() 
+    virtual void setNewCoords(int macroEl = -1) 
     {
       FUNCNAME("RefinementManager::setNewCoords");
       ERROR_EXIT("called for base class!\n");

AMDiS/src/RefinementManager1d.cc

@@ -138,7 +138,7 @@ namespace AMDiS {
     }
   }
 
-  void RefinementManager1d::setNewCoords()
+  void RefinementManager1d::setNewCoords(int)
   {
     TraverseStack stack;
     ElInfo *elInfo = stack.traverseFirst(mesh, -1, 

AMDiS/src/RefinementManager1d.h

@@ -43,7 +43,7 @@ namespace AMDiS {
     Flag refineMesh(Mesh *aMesh);
 
     /// Implements RefinementManager::setNewCoords
-    void setNewCoords();
+    void setNewCoords(int macroEl = -1);
 
   protected:
     /// Used by refineMesh while mesh traversal

AMDiS/src/RefinementManager2d.cc

@@ -23,6 +23,8 @@ namespace AMDiS {
     if (elInfo->getElement()->getMark() <= 0)
       return elInfo;    
 
+    MSG("RFCT 1\n");
+
     bool bound = false;
     DegreeOfFreedom *edge[2];
     RCNeighbourList* refineList = new RCNeighbourList(2);
@@ -114,12 +116,16 @@ namespace AMDiS {
   
     delete refineList;
 
+    MSG("RFCT 2\n");
+
     return elInfo;
   }
 
 
   void RefinementManager2d::newCoordsFct(ElInfo *elInfo)
   {
+    FUNCNAME("RefinementManager2d::newCoordsFct()");
+
     Element *el = elInfo->getElement();
     int dow = Global::getGeo(WORLD);
 
@@ -131,21 +137,33 @@ namespace AMDiS {
     if (el->getFirstChild() && projector && (!el->isNewCoordSet())) {
       WorldVector<double> *new_coord = new WorldVector<double>;
       
+      MSG("NU ABER 1\n");
       for (int j = 0; j < dow; j++)
 	(*new_coord)[j] = (elInfo->getCoord(0)[j] + elInfo->getCoord(1)[j]) * 0.5;
+      MSG("NU ABER 2\n");
       
       projector->project(*new_coord);
       el->setNewCoord(new_coord);
+
+      MSG("NU ABER 3\n");
     }
   }
 
 
-  void RefinementManager2d::setNewCoords()
+  void RefinementManager2d::setNewCoords(int macroEl)
   {
     TraverseStack stack;
-    ElInfo *elInfo = stack.traverseFirst(mesh, -1, 
-					 Mesh::CALL_EVERY_EL_PREORDER | 
-					 Mesh::FILL_BOUND | Mesh::FILL_COORDS);
+    ElInfo *elInfo;
+
+    if (macroEl == -1)
+      elInfo = stack.traverseFirst(mesh, -1, 
+				   Mesh::CALL_EVERY_EL_PREORDER | 
+				   Mesh::FILL_BOUND | Mesh::FILL_COORDS);
+    else
+      elInfo = stack.traverseFirstOneMacro(mesh, macroEl, -1,
+					   Mesh::CALL_EVERY_EL_PREORDER | 
+					   Mesh::FILL_BOUND | Mesh::FILL_COORDS);
+    
     while (elInfo) {
       newCoordsFct(elInfo);
       elInfo = stack.traverseNext(elInfo);

AMDiS/src/RefinementManager2d.h

@@ -44,7 +44,7 @@ namespace AMDiS {
     void newCoordsFct(ElInfo *elInfo);
 
     /// Implements RefinementManager::setNewCoords
-    void setNewCoords();
+    void setNewCoords(int macroEl = -1);
 
     /** \brief
      *  gets the elements around the refinement edge with vertices  node[0] and

AMDiS/src/RefinementManager3d.cc

@@ -285,17 +285,26 @@ namespace AMDiS {
   }
 
 
-  void RefinementManager3d::setNewCoords()
+  void RefinementManager3d::setNewCoords(int macroEl)
   {
     if (refList)
       delete refList;
  
     refList = new RCNeighbourList(mesh->getMaxEdgeNeigh());
+    ElInfo *elInfo;
+
+    if (macroEl == -1)
+      elInfo = stack->traverseFirst(mesh, -1, 
+				    Mesh::CALL_EVERY_EL_PREORDER | 
+				    Mesh::FILL_BOUND | Mesh::FILL_COORDS | 
+				    Mesh::FILL_NEIGH);
+    else
+      elInfo = stack->traverseFirstOneMacro(mesh, macroEl, -1,
+					    Mesh::CALL_EVERY_EL_PREORDER | 
+					    Mesh::FILL_BOUND | Mesh::FILL_COORDS | 
+					    Mesh::FILL_NEIGH);
+    
 
-    ElInfo *elInfo = stack->traverseFirst(mesh, -1, 
-					  Mesh::CALL_EVERY_EL_PREORDER | 
-					  Mesh::FILL_BOUND | Mesh::FILL_COORDS | 
-					  Mesh::FILL_NEIGH);
     while (elInfo) {
       newCoordsFct(elInfo);
       elInfo = stack->traverseNext(elInfo);

AMDiS/src/RefinementManager3d.h

@@ -44,7 +44,7 @@ namespace AMDiS {
     int newCoordsFct(ElInfo *el_info);
 
     /// Implements RefinementManager::setNewCoords
-    void setNewCoords();
+    void setNewCoords(int macroEl = -1);
 
     /** \brief
      * Gets the elements around the refinement edge with vertices node[0] and

AMDiS/src/parallel/MeshDistributor.cc

@@ -56,6 +56,7 @@ namespace AMDiS {
       deserialized(false),
       writeSerializationFile(false),
       repartitioningAllowed(false),
+      nTimestepsAfterLastRepartitioning(0),
       lastMeshChangeIndex(0)
   {
     FUNCNAME("MeshDistributor::ParalleDomainBase()");
@@ -547,8 +548,14 @@ namespace AMDiS {
 
     // === The mesh has changed, so check if it is required to repartition the mesh. ===
 
-    if (repartitioningAllowed)
-      repartitionMesh();
+    nTimestepsAfterLastRepartitioning++;
+
+    if (repartitioningAllowed) {
+      if (nTimestepsAfterLastRepartitioning >= 20) {
+	repartitionMesh();
+	nTimestepsAfterLastRepartitioning = 0;
+      }
+    }
   }
 
   
@@ -965,7 +972,7 @@ namespace AMDiS {
 
     DOFVector<double> tmpa(feSpace, "tmp");
     tmpa.set(mpiRank);
-    VtkWriter::writeFile(tmpa, "before-repartition.vtu");
+    VtkWriter::writeFile(testVec[0], "before-repartition.vtu");
 
     MSG("USED-SIZE A: %d\n", mesh->getDofAdmin(0).getUsedDofs());
 
@@ -985,6 +992,7 @@ namespace AMDiS {
     MacroInfo* minfo = mesh->getMacroFileInfo();
     TEST_EXIT_DBG(minfo)("No macro file info!\n");
 
+    MSG("MARK 1\n");
 
     // === Create map that maps macro element indices to pointers to the ===
     // === macro elements.                                               ===
@@ -994,7 +1002,7 @@ namespace AMDiS {
 	 it != minfo->mel.end(); ++it)
       elIndexMap[(*it)->getIndex()] = *it;
 
-    
+    MSG("MARK 2\n");    
     // === Create set of all new macro elements this rank will receive from ===
     // === other ranks.                                                     ===
 
@@ -1014,7 +1022,7 @@ namespace AMDiS {
 	 it != mesh->endOfMacroElements(); ++it)
       allMacroEl.insert(*it);
 
-
+    MSG("MARK 3\n");
     // === Add new macro elements to mesh. ===
 
     for (std::set<MacroElement*>::iterator it = newMacroEl.begin();
@@ -1029,7 +1037,7 @@ namespace AMDiS {
       mesh->getMacroElements().push_back(mel);
     }