Work on 3d parallelization.

AMDiS/src/DOFVector.hh

@@ -378,9 +378,9 @@ namespace AMDiS {
   template<typename T>
   void DOFVector<T>::interpol(AbstractFunction<T, WorldVector<double> > *fct)
   {
-    FUNCNAME("interpol");
+    FUNCNAME("DOFVector::interpol()");
 
-    TEST_EXIT_DBG(fct)("no function to interpolate\n");
+    TEST_EXIT_DBG(fct)("No function to interpolate!\n");
 
     interFct = fct;
 

AMDiS/src/ParallelDomainBase.h

@@ -193,11 +193,27 @@ namespace AMDiS {
 
   protected:
     /** \brief
-     * Determine the interior boundaries, i.e. boundaries between ranks, and store
+     * Determines the interior boundaries, i.e. boundaries between ranks, and stores
      * all information about them in \ref interiorBoundary.
      */
     void createInteriorBoundaryInfo();
 
+    /** \brief
+     * Deterimes the interior boundaries between ranks, that are based on the 
+     * neighbourhood information of the macro elements. That means that in 2d the
+     * function search for all edge based interior boundaries and in 3d for all face
+     * based interior boundaries. This function cannot find boundaries of substructure
+     * elements, i.e. vertex boundaries in 2d and vertex and edge boundaries in 3d.
+     */
+    void createMacroElementInteriorBoundaryInfo();
+
+    /** \brief
+     * Determines all interior boundaries between rank that consist of element's
+     * substructures. In 2d these may be only vertices. In 3d there may be also
+     * interior boundaries consisting of either a whole edge or of a single vertex.
+     */
+    void createSubstructureInteriorBoundaryInfo();
+
     /// Removes all macro elements from the mesh that are not part of ranks partition.
     void removeMacroElements();
 
@@ -342,16 +358,23 @@ namespace AMDiS {
     void writePartitioningMesh(std::string filename);
 
     /** \brief
-     * This function must be used if the values of a SystemVector must be synchronised
+     * This function must be used if the values of a DOFVector must be synchronised
      * over all ranks. That means, that each rank sends the values of the DOFs, which
      * are owned by the rank and lie on an interior bounday, to all other ranks also
-     * having this DOF.
+     * having these DOFs.
      *
      * This function must be used, for example, after the lineary system is solved, or
-     * after the SystemVector is set by some user defined functions, e.g., initial
+     * after the DOFVector is set by some user defined functions, e.g., initial
      * solution functions.
+     */    
+    void synchVector(DOFVector<double> &vec);
+
+    /** \brief
+     * Works in the same way as the function above defined for DOFVectors. Due to
+     * performance, this function does not call \ref synchVector for each DOFVector,
+     * but instead sends all values of all DOFVectors all at once.
      */
-    void synchVectors(SystemVector &vec);
+    void synchVector(SystemVector &vec);
 
     /// Writes a vector of dof pointers to an output stream.
     void serialize(std::ostream &out, DofContainer &data);

AMDiS/src/ParallelDomainVec.cc

@@ -103,7 +103,7 @@ namespace AMDiS {
   {   
     ParallelDomainBase::solveInitialProblem(adaptInfo);
     
-    synchVectors(*(probVec->getSolution()));
+    synchVector(*(probVec->getSolution()));
   }
 
   void ParallelDomainVec::solve()

AMDiS/src/ProblemVec.cc

 #include <sstream>
-#include "boost/lexical_cast.hpp"
+#include <boost/lexical_cast.hpp>
 #include "ProblemVec.h"
 #include "RecoveryEstimator.h"
 #include "Serializer.h"

AMDiS/src/StdMpi.h

@@ -48,6 +48,16 @@ namespace AMDiS {
     return data.size() * Global::getGeo(WORLD);
   }
 
+  int intSizeOf(std::vector<int> &data)
+  {
+    return data.size();
+  }
+
+  int intSizeOf(std::vector<double> &data)
+  {
+    return data.size();
+  }
+  
   int intSizeOf(std::vector<std::pair<int, int> > &data)
   {
     return data.size() * 2;
@@ -146,6 +156,34 @@ namespace AMDiS {
 	data[i][j] = buf[pos++];
   }
 
+  void makeBuf(std::vector<int> &data, int *buf)
+  {
+    for (unsigned int i = 0; i < data.size(); i++)
+      buf[i] = data[i];
+  }
+
+  void makeFromBuf(std::vector<int> &data, int *buf, int bufSize)
+  {
+    data.resize(bufSize);
+
+    for (int i = 0; i < bufSize; i++)
+      data[i] = buf[i];
+  }
+
+  void makeBuf(std::vector<double> &data, double *buf)
+  {
+    for (unsigned int i = 0; i < data.size(); i++)
+      buf[i] = data[i];
+  }
+
+  void makeFromBuf(std::vector<double> &data, double *buf, int bufSize)
+  {
+    data.resize(bufSize);
+
+    for (int i = 0; i < bufSize; i++)
+      data[i] = buf[i];
+  }
+
   void makeBuf(std::vector<std::pair<int, int> > &data, int *buf)
   {
     for (unsigned int i = 0; i < data.size(); i++) {
@@ -206,7 +244,7 @@ namespace AMDiS {
     }
   }
 
-  template<typename SendT, typename RecvT>
+  template<typename SendT, typename RecvT=SendT>
   class StdMpi 
   {
   public:

AMDiS/src/Tetrahedron.cc

@@ -193,6 +193,9 @@ namespace AMDiS {
     FUNCNAME("Tetrahedron::getVertexDofs()");
 
     switch (bound.subObj) {
+    case VERTEX:
+      dofs.push_back(dof[bound.ithObj]);
+      break;
     case EDGE:
       getVertexDofsAtEdge(feSpace, bound, dofs, parentVertices);
       break;

AMDiS/src/VtkWriter.cc

@@ -5,6 +5,10 @@
 #include <cmath>
 #include <boost/filesystem/operations.hpp>
 #include <boost/filesystem/convenience.hpp>
+#include <boost/lexical_cast.hpp>
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+#include <mpi.h>
+#endif
 
 #include "VtkWriter.h"
 #include "DataCollector.h"
@@ -125,16 +129,27 @@ namespace AMDiS {
 
   void VtkWriter::writeFile(DOFVector<double> *values, std::string filename)
   {
-    DataCollector *dc = new DataCollector(values->getFESpace(), values);
+    FUNCNAME("VtkWriter::writeFile()");
 
+    DataCollector dc(values->getFESpace(), values);
     std::vector<DataCollector*> dcList(0);
-    dcList.push_back(dc);
+    dcList.push_back(&dc);
+    VtkWriter writer(&dcList);
+
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+    using boost::lexical_cast;
+  
+    int sPos = filename.find(".vtu");
+    TEST_EXIT(sPos >= 0)("Failed to find file postfix!\n");
+    std::string name = filename.substr(0, sPos);
 
-    VtkWriter *writer = new VtkWriter(&dcList);
-    writer->writeFile(filename);
+    if (MPI::COMM_WORLD.Get_rank() == 0)
+      writer.writeParallelFile(name + ".pvtu", MPI::COMM_WORLD.Get_size(), name, ".vtu");    
+
+    filename = name + "-p" + lexical_cast<std::string>(MPI::COMM_WORLD.Get_rank()) + "-.vtu";
+#endif
 
-    delete writer;
-    delete dc;
+    writer.writeFile(filename);
   }
 
 }