Commit be06847b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work in progress on parallel domain decomposition.

parent 29f288f1
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include "Projection.h" #include "Projection.h"
namespace AMDiS { namespace AMDiS {
DataCollector::DataCollector(const FiniteElemSpace *feSpace, DataCollector::DataCollector(const FiniteElemSpace *feSpace,
DOFVector<double> *values, DOFVector<double> *values,
int level, int level,
...@@ -61,20 +62,17 @@ namespace AMDiS { ...@@ -61,20 +62,17 @@ namespace AMDiS {
void DataCollector::fillAllData() void DataCollector::fillAllData()
{ {
if (!elementDataCollected_) { if (!elementDataCollected_)
startCollectingElementData(); startCollectingElementData();
}
if (!periodicDataCollected_) { if (!periodicDataCollected_)
startCollectingPeriodicData(); startCollectingPeriodicData();
}
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
} }
int DataCollector::startCollectingElementData() void DataCollector::startCollectingElementData()
{ {
FUNCNAME("DataCollector::startCollectingElementData()"); FUNCNAME("DataCollector::startCollectingElementData()");
...@@ -99,26 +97,24 @@ namespace AMDiS { ...@@ -99,26 +97,24 @@ namespace AMDiS {
// Traverse elements to create element information // Traverse elements to create element information
elInfo = stack.traverseFirst(mesh_, level_, flag); elInfo = stack.traverseFirst(mesh_, level_, flag);
while (elInfo) { while (elInfo) {
if (!writeElem_ || writeElem_(elInfo)) { if (!writeElem_ || writeElem_(elInfo))
addElementData(elInfo); addElementData(elInfo);
}
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
elementDataCollected_ = true; elementDataCollected_ = true;
return(0);
} }
int DataCollector::startCollectingValueData() void DataCollector::startCollectingValueData()
{ {
FUNCNAME("DataCollector::startCollectingValueData()"); FUNCNAME("DataCollector::startCollectingValueData()");
DOFVector<int>::Iterator intPointIt(interpPointInd_, USED_DOFS); DOFVector<int>::Iterator intPointIt(interpPointInd_, USED_DOFS);
for (intPointIt.reset(); !intPointIt.end(); ++intPointIt) { for (intPointIt.reset(); !intPointIt.end(); ++intPointIt)
(*intPointIt) = -1; (*intPointIt) = -1;
}
interpPoints_.clear(); interpPoints_.clear();
...@@ -130,13 +126,11 @@ namespace AMDiS { ...@@ -130,13 +126,11 @@ namespace AMDiS {
// Traverse elements to add value information and to mark // Traverse elements to add value information and to mark
// interpolation points. // interpolation points.
ElInfo *elInfo = stack.traverseFirst(mesh_, ElInfo *elInfo = stack.traverseFirst(mesh_, level_,
level_,
traverseFlag_ | Mesh::FILL_COORDS); traverseFlag_ | Mesh::FILL_COORDS);
while (elInfo) { while (elInfo) {
if (!writeElem_ || writeElem_(elInfo)) { if (!writeElem_ || writeElem_(elInfo))
addValueData(elInfo); addValueData(elInfo);
}
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
...@@ -146,11 +140,9 @@ namespace AMDiS { ...@@ -146,11 +140,9 @@ namespace AMDiS {
// Remove all interpolation marks and, instead, set to each // Remove all interpolation marks and, instead, set to each
// interpolation point its continous index starting from 0. // interpolation point its continous index starting from 0.
int i = 0; int i = 0;
for (intPointIt.reset(); !intPointIt.end(); ++intPointIt) { for (intPointIt.reset(); !intPointIt.end(); ++intPointIt)
if (*intPointIt == -3) { if (*intPointIt == -3)
*intPointIt = i++; *intPointIt = i++;
}
}
// Traverse elements to create interpolation values. // Traverse elements to create interpolation values.
elInfo = stack.traverseFirst(mesh_, level_, traverseFlag_ | Mesh::FILL_COORDS); elInfo = stack.traverseFirst(mesh_, level_, traverseFlag_ | Mesh::FILL_COORDS);
...@@ -163,11 +155,9 @@ namespace AMDiS { ...@@ -163,11 +155,9 @@ namespace AMDiS {
delete [] localDOFs_; delete [] localDOFs_;
valueDataCollected_ = true; valueDataCollected_ = true;
return 0;
} }
int DataCollector::startCollectingPeriodicData() void DataCollector::startCollectingPeriodicData()
{ {
FUNCNAME("DataCollector::startCollectingPeriodicData()"); FUNCNAME("DataCollector::startCollectingPeriodicData()");
...@@ -210,11 +200,9 @@ namespace AMDiS { ...@@ -210,11 +200,9 @@ namespace AMDiS {
} }
periodicDataCollected_ = true; periodicDataCollected_ = true;
return(0);
} }
int DataCollector::addElementData(ElInfo* elInfo) void DataCollector::addElementData(ElInfo* elInfo)
{ {
FUNCNAME("DataCollector::addElementData()"); FUNCNAME("DataCollector::addElementData()");
...@@ -280,19 +268,15 @@ namespace AMDiS { ...@@ -280,19 +268,15 @@ namespace AMDiS {
outputIndices_[elInfo->getNeighbour(i)->getIndex()] : outputIndices_[elInfo->getNeighbour(i)->getIndex()] :
-1; -1;
} }
if (dim_ == 3) { if (dim_ == 3)
elementInfo.type = (dynamic_cast<ElInfo3d*>(elInfo)->getType()); elementInfo.type = (dynamic_cast<ElInfo3d*>(elInfo)->getType());
}
// remember element info // remember element info
elements_.push_back(elementInfo); elements_.push_back(elementInfo);
return(0);
} }
int DataCollector::addValueData(ElInfo *elInfo) void DataCollector::addValueData(ElInfo *elInfo)
{ {
FUNCNAME("DataCollector::addValueData()"); FUNCNAME("DataCollector::addValueData()");
...@@ -345,28 +329,24 @@ namespace AMDiS { ...@@ -345,28 +329,24 @@ namespace AMDiS {
nInterpPoints_++; nInterpPoints_++;
} }
} }
} }
return(0);
} }
int DataCollector::addInterpData(ElInfo *elInfo) void DataCollector::addInterpData(ElInfo *elInfo)
{ {
FUNCNAME("DataCollector::addInterpData()"); FUNCNAME("DataCollector::addInterpData()");
basisFcts_->getLocalIndices(elInfo->getElement(), localAdmin_, localDOFs_); basisFcts_->getLocalIndices(elInfo->getElement(), localAdmin_, localDOFs_);
std::vector<int> elemInterpPoints(0); std::vector<int> elemInterpPoints(0);
for (int i = mesh_->getGeo(VERTEX); i < nBasisFcts_; i++) { for (int i = mesh_->getGeo(VERTEX); i < nBasisFcts_; i++)
elemInterpPoints.push_back((*interpPointInd_)[localDOFs_[i]]); elemInterpPoints.push_back((*interpPointInd_)[localDOFs_[i]]);
}
interpPoints_.push_back(elemInterpPoints); interpPoints_.push_back(elemInterpPoints);
return(0);
} }
int DataCollector::addPeriodicData(ElInfo *elInfo) { void DataCollector::addPeriodicData(ElInfo *elInfo)
{
FUNCNAME("DataCollector::addPeriodicData"); FUNCNAME("DataCollector::addPeriodicData");
LeafDataPeriodic *ldp = dynamic_cast<LeafDataPeriodic*> LeafDataPeriodic *ldp = dynamic_cast<LeafDataPeriodic*>
...@@ -423,69 +403,60 @@ namespace AMDiS { ...@@ -423,69 +403,60 @@ namespace AMDiS {
} }
} }
} }
return(0);
} }
std::list<ElementInfo>* DataCollector::getElementInfos() std::list<ElementInfo>* DataCollector::getElementInfos()
{ {
if (!elementDataCollected_) { if (!elementDataCollected_)
startCollectingElementData(); startCollectingElementData();
}
return &elements_; return &elements_;
} }
DOFVector< std::list<VertexInfo> >* DataCollector::getVertexInfos() DOFVector< std::list<VertexInfo> >* DataCollector::getVertexInfos()
{ {
if (!elementDataCollected_) { if (!elementDataCollected_)
startCollectingElementData(); startCollectingElementData();
}
return vertexInfos_; return vertexInfos_;
} }
std::list<PeriodicInfo>* DataCollector::getPeriodicInfos() std::list<PeriodicInfo>* DataCollector::getPeriodicInfos()
{ {
if (!periodicDataCollected_) { if (!periodicDataCollected_)
startCollectingPeriodicData(); startCollectingPeriodicData();
}
return &periodicInfos_; return &periodicInfos_;
} }
int DataCollector::getNumberVertices() int DataCollector::getNumberVertices()
{ {
if (!elementDataCollected_) { if (!elementDataCollected_)
startCollectingElementData(); startCollectingElementData();
}
return nVertices_; return nVertices_;
} }
int DataCollector::getNumberElements() int DataCollector::getNumberElements()
{ {
if (!elementDataCollected_) { if (!elementDataCollected_)
startCollectingElementData(); startCollectingElementData();
}
return nElements_; return nElements_;
} }
int DataCollector::getNumberInterpPoints() int DataCollector::getNumberInterpPoints()
{ {
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
return nInterpPoints_; return nInterpPoints_;
} }
int DataCollector::getNumberConnections() int DataCollector::getNumberConnections()
{ {
if (!periodicDataCollected_) { if (!periodicDataCollected_)
startCollectingPeriodicData(); startCollectingPeriodicData();
}
return nConnections_; return nConnections_;
} }
...@@ -502,45 +473,40 @@ namespace AMDiS { ...@@ -502,45 +473,40 @@ namespace AMDiS {
DOFVector<double>* DataCollector::getValues() DOFVector<double>* DataCollector::getValues()
{ {
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
return values_; return values_;
} }
DOFVector< std::list<WorldVector<double> > >* DataCollector::getDofCoords() DOFVector< std::list<WorldVector<double> > >* DataCollector::getDofCoords()
{ {
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
return dofCoords_; return dofCoords_;
} }
DOFVector<int>* DataCollector::getInterpPointInd() DOFVector<int>* DataCollector::getInterpPointInd()
{ {
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
return interpPointInd_; return interpPointInd_;
} }
DOFVector< std::list<WorldVector<double> > >* DataCollector::getInterpPointCoords() DOFVector< std::list<WorldVector<double> > >* DataCollector::getInterpPointCoords()
{ {
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
return interpPointCoords_; return interpPointCoords_;
} }
std::vector< std::vector<int> >* DataCollector::getInterpPoints() std::vector< std::vector<int> >* DataCollector::getInterpPoints()
{ {
if (!valueDataCollected_) { if (!valueDataCollected_)
startCollectingValueData(); startCollectingValueData();
}
return &interpPoints_; return &interpPoints_;
} }
......
...@@ -101,25 +101,25 @@ namespace AMDiS { ...@@ -101,25 +101,25 @@ namespace AMDiS {
protected: protected:
/// Start collecting element and vertex data of the problem. /// Start collecting element and vertex data of the problem.
int startCollectingElementData(); void startCollectingElementData();
/// Start collecting value data of the problem. /// Start collecting value data of the problem.
int startCollectingValueData(); void startCollectingValueData();
/// Start collecting periodic data of the problem. /// Start collecting periodic data of the problem.
int startCollectingPeriodicData(); void startCollectingPeriodicData();
/// Adds information about one element and its vertices. /// Adds information about one element and its vertices.
int addElementData(ElInfo* elInfo); void addElementData(ElInfo* elInfo);
/// Adds value information of one element. /// Adds value information of one element.
int addValueData(ElInfo *elInfo); void addValueData(ElInfo *elInfo);
/// Adds information about interpolation points of vertices. /// Adds information about interpolation points of vertices.
int addInterpData(ElInfo *elInfo); void addInterpData(ElInfo *elInfo);
/// Adds value information of one element. /// Adds value information of one element.
int addPeriodicData(ElInfo *elInfo); void addPeriodicData(ElInfo *elInfo);
/// Vector with vertex values /// Vector with vertex values
DOFVector<double> *values_; DOFVector<double> *values_;
......
...@@ -10,6 +10,9 @@ ...@@ -10,6 +10,9 @@
#include "PartitionElementData.h" #include "PartitionElementData.h"
#include "DOFMatrix.h" #include "DOFMatrix.h"
#include "DOFVector.h" #include "DOFVector.h"
#include "VtkWriter.h"
#include "petscksp.h"
namespace AMDiS { namespace AMDiS {
...@@ -138,9 +141,8 @@ namespace AMDiS { ...@@ -138,9 +141,8 @@ namespace AMDiS {
PartitionElementData *partitionData = PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*> dynamic_cast<PartitionElementData*>
((*it)->getElement()->getElementData(PARTITION_ED)); ((*it)->getElement()->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN) { if (partitionData->getPartitionStatus() != IN)
macrosToRemove.push_back(*it); macrosToRemove.push_back(*it);
}
} }
mesh->removeMacroElements(macrosToRemove); mesh->removeMacroElements(macrosToRemove);
...@@ -151,17 +153,15 @@ namespace AMDiS { ...@@ -151,17 +153,15 @@ namespace AMDiS {
int *lOrder = (int*)(malloc(sizeof(int) * rankDofs.size())); int *lOrder = (int*)(malloc(sizeof(int) * rankDofs.size()));
for (std::vector<const DegreeOfFreedom*>::iterator it = rankDofs.begin(); for (std::vector<const DegreeOfFreedom*>::iterator it = rankDofs.begin();
it != rankDofs.end(); ++it) { it != rankDofs.end(); ++it)
gOrder[nRankDOFs++] = (*it)[0]; gOrder[nRankDOFs++] = (*it)[0];
}
int rstart = 0; int rstart = 0;
MPI_Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD); MPI_Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
rstart -= nRankDOFs; rstart -= nRankDOFs;
for (int i = 0; i < nRankDOFs; i++) { for (int i = 0; i < nRankDOFs; i++)
lOrder[i] = rstart + i; lOrder[i] = rstart + i;
}
AOCreateBasic(PETSC_COMM_WORLD, nRankDOFs, gOrder, lOrder, &applicationOrdering); AOCreateBasic(PETSC_COMM_WORLD, nRankDOFs, gOrder, lOrder, &applicationOrdering);
...@@ -275,6 +275,10 @@ namespace AMDiS { ...@@ -275,6 +275,10 @@ namespace AMDiS {
ierr = VecCreate(PETSC_COMM_WORLD, &petscRhsVec); ierr = VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
ierr = VecSetSizes(petscRhsVec, rankDofs.size(), partitionDofs.size()); ierr = VecSetSizes(petscRhsVec, rankDofs.size(), partitionDofs.size());
ierr = VecSetType(petscRhsVec, VECMPI); ierr = VecSetType(petscRhsVec, VECMPI);
ierr = VecCreate(PETSC_COMM_WORLD, &petscSolVec);
ierr = VecSetSizes(petscSolVec, rankDofs.size(), partitionDofs.size());
ierr = VecSetType(petscSolVec, VECMPI);
} }
void ParallelDomainProblemBase::exitParallelization(AdaptInfo *adaptInfo) void ParallelDomainProblemBase::exitParallelization(AdaptInfo *adaptInfo)
...@@ -285,14 +289,27 @@ namespace AMDiS { ...@@ -285,14 +289,27 @@ namespace AMDiS {
void ParallelDomainProblemBase::fillPetscMatrix(DOFMatrix *mat, void ParallelDomainProblemBase::fillPetscMatrix(DOFMatrix *mat,
DOFVector<double> *vec) DOFVector<double> *vec)
{ {
/* DOFMatrix::Iterator rowIt(mat, USED_DOFS); using mtl::tag::major; using mtl::tag::nz; using mtl::begin; using mtl::end;
for (rowIt.reset(); !rowIt.end(); ++rowIt) { namespace traits= mtl::traits;
for (int i = 0; i < static_cast<int>((*rowIt).size()); i++) { typedef DOFMatrix::base_matrix_type Matrix;
if ((*rowIt)[i].col >= 0) {
MatSetValues(petscMatrix, 1, &i, 1, &((*rowIt)[i].col), &((*rowIt)[i].entry), ADD_VALUES); traits::row<Matrix>::type row(mat->getBaseMatrix());
traits::col<Matrix>::type col(mat->getBaseMatrix());
traits::const_value<Matrix>::type value(mat->getBaseMatrix());
typedef traits::range_generator<major, Matrix>::type cursor_type;
typedef traits::range_generator<nz, cursor_type>::type icursor_type;
std::cout.precision(10);
for (cursor_type cursor = begin<major>(mat->getBaseMatrix()), cend = end<major>(mat->getBaseMatrix()); cursor != cend; ++cursor)
for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); icursor != icend; ++icursor)
if (value(*icursor) != 0.0) {
int r = row(*icursor);
int c = col(*icursor);
double v = value(*icursor);
MatSetValues(petscMatrix, 1, &r, 1, &c, &v, ADD_VALUES);
} }
}
}
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
...@@ -303,7 +320,38 @@ namespace AMDiS { ...@@ -303,7 +320,38 @@ namespace AMDiS {
double value = *dofIt; double value = *dofIt;
VecSetValues(petscRhsVec, 1, &index, &value, ADD_VALUES); VecSetValues(petscRhsVec, 1, &index, &value, ADD_VALUES);
}*/ }
}
void ParallelDomainProblemBase::solvePetscMatrix(DOFVector<double> *vec)
{
KSP ksp;
PC pc;