Skip to content

GitLab

Commit be06847b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work in progress on parallel domain decomposition.

parent 29f288f1
Hide whitespace changes
Inline Side-by-side
Showing with 128 additions and 114 deletions
AMDiS/src/DataCollector.cc
View file @ be06847b
......@@ -6,6 +6,7 @@
#include "Projection.h"
namespace AMDiS {
DataCollector::DataCollector(const FiniteElemSpace *feSpace,
DOFVector<double> *values,
int level,
......@@ -61,20 +62,17 @@ namespace AMDiS {
void DataCollector::fillAllData()
{
if (!elementDataCollected_) {
if (!elementDataCollected_)
startCollectingElementData();
}
if (!periodicDataCollected_) {
if (!periodicDataCollected_)
startCollectingPeriodicData();
}
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
}
int DataCollector::startCollectingElementData()
void DataCollector::startCollectingElementData()
{
FUNCNAME("DataCollector::startCollectingElementData()");
......@@ -99,26 +97,24 @@ namespace AMDiS {
// Traverse elements to create element information
elInfo = stack.traverseFirst(mesh_, level_, flag);
while (elInfo) {
if (!writeElem_ || writeElem_(elInfo)) {
if (!writeElem_ || writeElem_(elInfo))
addElementData(elInfo);
}
elInfo = stack.traverseNext(elInfo);
}
elementDataCollected_ = true;
return(0);
}
int DataCollector::startCollectingValueData()
void DataCollector::startCollectingValueData()
{
FUNCNAME("DataCollector::startCollectingValueData()");
DOFVector<int>::Iterator intPointIt(interpPointInd_, USED_DOFS);
for (intPointIt.reset(); !intPointIt.end(); ++intPointIt) {
for (intPointIt.reset(); !intPointIt.end(); ++intPointIt)
(*intPointIt) = -1;
}
interpPoints_.clear();
......@@ -130,13 +126,11 @@ namespace AMDiS {
// Traverse elements to add value information and to mark
// interpolation points.
ElInfo *elInfo = stack.traverseFirst(mesh_,
level_,
ElInfo *elInfo = stack.traverseFirst(mesh_, level_,
traverseFlag_ | Mesh::FILL_COORDS);
while (elInfo) {
if (!writeElem_ || writeElem_(elInfo)) {
if (!writeElem_ || writeElem_(elInfo))
addValueData(elInfo);
}
elInfo = stack.traverseNext(elInfo);
}
......@@ -146,11 +140,9 @@ namespace AMDiS {
// Remove all interpolation marks and, instead, set to each
// interpolation point its continous index starting from 0.
int i = 0;
for (intPointIt.reset(); !intPointIt.end(); ++intPointIt) {
if (*intPointIt == -3) {
for (intPointIt.reset(); !intPointIt.end(); ++intPointIt)
if (*intPointIt == -3)
*intPointIt = i++;
}
}
// Traverse elements to create interpolation values.
elInfo = stack.traverseFirst(mesh_, level_, traverseFlag_ | Mesh::FILL_COORDS);
......@@ -163,11 +155,9 @@ namespace AMDiS {
delete [] localDOFs_;
valueDataCollected_ = true;
return 0;
}
int DataCollector::startCollectingPeriodicData()
void DataCollector::startCollectingPeriodicData()
{
FUNCNAME("DataCollector::startCollectingPeriodicData()");
......@@ -210,11 +200,9 @@ namespace AMDiS {
}
periodicDataCollected_ = true;
return(0);
}
int DataCollector::addElementData(ElInfo* elInfo)
void DataCollector::addElementData(ElInfo* elInfo)
{
FUNCNAME("DataCollector::addElementData()");
......@@ -280,19 +268,15 @@ namespace AMDiS {
outputIndices_[elInfo->getNeighbour(i)->getIndex()] :
-1;
}
if (dim_ == 3) {
if (dim_ == 3)
elementInfo.type = (dynamic_cast<ElInfo3d*>(elInfo)->getType());
}
// remember element info
elements_.push_back(elementInfo);
return(0);
}
int DataCollector::addValueData(ElInfo *elInfo)
void DataCollector::addValueData(ElInfo *elInfo)
{
FUNCNAME("DataCollector::addValueData()");
......@@ -345,28 +329,24 @@ namespace AMDiS {
nInterpPoints_++;
}
}
}
return(0);
}
}
int DataCollector::addInterpData(ElInfo *elInfo)
void DataCollector::addInterpData(ElInfo *elInfo)
{
FUNCNAME("DataCollector::addInterpData()");
basisFcts_->getLocalIndices(elInfo->getElement(), localAdmin_, localDOFs_);
std::vector<int> elemInterpPoints(0);
for (int i = mesh_->getGeo(VERTEX); i < nBasisFcts_; i++) {
for (int i = mesh_->getGeo(VERTEX); i < nBasisFcts_; i++)
elemInterpPoints.push_back((*interpPointInd_)[localDOFs_[i]]);
}
interpPoints_.push_back(elemInterpPoints);
return(0);
interpPoints_.push_back(elemInterpPoints);
}
int DataCollector::addPeriodicData(ElInfo *elInfo) {
void DataCollector::addPeriodicData(ElInfo *elInfo)
{
FUNCNAME("DataCollector::addPeriodicData");
LeafDataPeriodic *ldp = dynamic_cast<LeafDataPeriodic*>
......@@ -423,69 +403,60 @@ namespace AMDiS {
}
}
}
return(0);
}
std::list<ElementInfo>* DataCollector::getElementInfos()
{
if (!elementDataCollected_) {
if (!elementDataCollected_)
startCollectingElementData();
}
return &elements_;
}
DOFVector< std::list<VertexInfo> >* DataCollector::getVertexInfos()
{
if (!elementDataCollected_) {
if (!elementDataCollected_)
startCollectingElementData();
}
return vertexInfos_;
}
std::list<PeriodicInfo>* DataCollector::getPeriodicInfos()
{
if (!periodicDataCollected_) {
if (!periodicDataCollected_)
startCollectingPeriodicData();
}
return &periodicInfos_;
}
int DataCollector::getNumberVertices()
{
if (!elementDataCollected_) {
if (!elementDataCollected_)
startCollectingElementData();
}
return nVertices_;
}
int DataCollector::getNumberElements()
{
if (!elementDataCollected_) {
if (!elementDataCollected_)
startCollectingElementData();
}
return nElements_;
}
int DataCollector::getNumberInterpPoints()
{
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
return nInterpPoints_;
}
int DataCollector::getNumberConnections()
{
if (!periodicDataCollected_) {
if (!periodicDataCollected_)
startCollectingPeriodicData();
}
return nConnections_;
}
......@@ -502,45 +473,40 @@ namespace AMDiS {
DOFVector<double>* DataCollector::getValues()
{
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
return values_;
}
DOFVector< std::list<WorldVector<double> > >* DataCollector::getDofCoords()
{
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
return dofCoords_;
}
DOFVector<int>* DataCollector::getInterpPointInd()
{
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
return interpPointInd_;
}
DOFVector< std::list<WorldVector<double> > >* DataCollector::getInterpPointCoords()
{
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
return interpPointCoords_;
}
std::vector< std::vector<int> >* DataCollector::getInterpPoints()
{
if (!valueDataCollected_) {
if (!valueDataCollected_)
startCollectingValueData();
}
return &interpPoints_;
}
......
AMDiS/src/DataCollector.h
View file @ be06847b
......@@ -101,25 +101,25 @@ namespace AMDiS {
protected:
/// Start collecting element and vertex data of the problem.
int startCollectingElementData();
void startCollectingElementData();
/// Start collecting value data of the problem.
int startCollectingValueData();
void startCollectingValueData();
/// Start collecting periodic data of the problem.
int startCollectingPeriodicData();
void startCollectingPeriodicData();
/// Adds information about one element and its vertices.
int addElementData(ElInfo* elInfo);
void addElementData(ElInfo* elInfo);
/// Adds value information of one element.
int addValueData(ElInfo *elInfo);
void addValueData(ElInfo *elInfo);
/// Adds information about interpolation points of vertices.
int addInterpData(ElInfo *elInfo);
void addInterpData(ElInfo *elInfo);
/// Adds value information of one element.
int addPeriodicData(ElInfo *elInfo);
void addPeriodicData(ElInfo *elInfo);
/// Vector with vertex values
DOFVector<double> *values_;
......
AMDiS/src/ParallelDomainProblem.cc
View file @ be06847b
......@@ -10,6 +10,9 @@
#include "PartitionElementData.h"
#include "DOFMatrix.h"
#include "DOFVector.h"
#include "VtkWriter.h"
#include "petscksp.h"
namespace AMDiS {
......@@ -138,9 +141,8 @@ namespace AMDiS {
PartitionElementData *partitionData =
dynamic_cast<PartitionElementData*>
((*it)->getElement()->getElementData(PARTITION_ED));
if (partitionData->getPartitionStatus() != IN) {
if (partitionData->getPartitionStatus() != IN)
macrosToRemove.push_back(*it);
}
}
mesh->removeMacroElements(macrosToRemove);
......@@ -151,17 +153,15 @@ namespace AMDiS {
int *lOrder = (int*)(malloc(sizeof(int) * rankDofs.size()));
for (std::vector<const DegreeOfFreedom*>::iterator it = rankDofs.begin();
it != rankDofs.end(); ++it) {
it != rankDofs.end(); ++it)
gOrder[nRankDOFs++] = (*it)[0];
}
int rstart = 0;
MPI_Scan(&nRankDOFs, &rstart, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
rstart -= nRankDOFs;
for (int i = 0; i < nRankDOFs; i++) {
for (int i = 0; i < nRankDOFs; i++)
lOrder[i] = rstart + i;
}
AOCreateBasic(PETSC_COMM_WORLD, nRankDOFs, gOrder, lOrder, &applicationOrdering);
......@@ -275,6 +275,10 @@ namespace AMDiS {
ierr = VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
ierr = VecSetSizes(petscRhsVec, rankDofs.size(), partitionDofs.size());
ierr = VecSetType(petscRhsVec, VECMPI);
ierr = VecCreate(PETSC_COMM_WORLD, &petscSolVec);
ierr = VecSetSizes(petscSolVec, rankDofs.size(), partitionDofs.size());
ierr = VecSetType(petscSolVec, VECMPI);
}
void ParallelDomainProblemBase::exitParallelization(AdaptInfo *adaptInfo)
......@@ -285,14 +289,27 @@ namespace AMDiS {
void ParallelDomainProblemBase::fillPetscMatrix(DOFMatrix *mat,
DOFVector<double> *vec)
{
/* DOFMatrix::Iterator rowIt(mat, USED_DOFS);
for (rowIt.reset(); !rowIt.end(); ++rowIt) {
for (int i = 0; i < static_cast<int>((*rowIt).size()); i++) {
if ((*rowIt)[i].col >= 0) {
MatSetValues(petscMatrix, 1, &i, 1, &((*rowIt)[i].col), &((*rowIt)[i].entry), ADD_VALUES);
using mtl::tag::major; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits= mtl::traits;
typedef DOFMatrix::base_matrix_type Matrix;
traits::row<Matrix>::type row(mat->getBaseMatrix());
traits::col<Matrix>::type col(mat->getBaseMatrix());
traits::const_value<Matrix>::type value(mat->getBaseMatrix());
typedef traits::range_generator<major, Matrix>::type cursor_type;
typedef traits::range_generator<nz, cursor_type>::type icursor_type;
std::cout.precision(10);
for (cursor_type cursor = begin<major>(mat->getBaseMatrix()), cend = end<major>(mat->getBaseMatrix()); cursor != cend; ++cursor)
for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor); icursor != icend; ++icursor)
if (value(*icursor) != 0.0) {
int r = row(*icursor);
int c = col(*icursor);
double v = value(*icursor);
MatSetValues(petscMatrix, 1, &r, 1, &c, &v, ADD_VALUES);
}
}
}
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(petscMatrix, MAT_FINAL_ASSEMBLY);
......@@ -303,7 +320,38 @@ namespace AMDiS {
double value = *dofIt;
VecSetValues(petscRhsVec, 1, &index, &value, ADD_VALUES);
}*/
}
}
void ParallelDomainProblemBase::solvePetscMatrix(DOFVector<double> *vec)
{
KSP ksp;
PC pc;
KSPCreate(PETSC_COMM_WORLD, &ksp);
KSPSetOperators(ksp, petscMatrix, petscMatrix, DIFFERENT_NONZERO_PATTERN);
KSPGetPC(ksp, &pc);
PCSetType(pc, PCJACOBI);
KSPSetTolerances(ksp, 1.e-7, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT);
KSPSetType(ksp, KSPBCGS);
KSPMonitorSet(ksp, KSPMonitorDefault, PETSC_NULL, 0);
KSPSolve(ksp, petscRhsVec, petscSolVec);
PetscScalar *vecPointer;
VecGetArray(petscSolVec, &vecPointer);
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
int counter = 0;
for (dofIt.reset(); !dofIt.end(); ++dofIt)
*vec = vecPointer[counter++];
VecRestoreArray(petscSolVec, &vecPointer);
std::stringstream oss;
oss << "test" << MPI::COMM_WORLD.Get_rank() << ".vtu";
VtkWriter::writeFile(vec, oss.str());
std::cout << "USED SIZE = " << vec->getUsedSize() << std::endl;
}
double ParallelDomainProblemBase::setElemWeights(AdaptInfo *adaptInfo)
......@@ -370,6 +418,8 @@ namespace AMDiS {
fillPetscMatrix(probScal->getSystemMatrix(), probScal->getRHS());
solvePetscMatrix(probScal->getSolution());
// if (toDo.isSet(SOLVE))
// iterationIF->getProblem()->solve(adaptInfo, false);
......
AMDiS/src/ParallelDomainProblem.h
View file @ be06847b
......@@ -105,6 +105,8 @@ namespace AMDiS {
void fillPetscMatrix(DOFMatrix *mat, DOFVector<double> *vec);
void solvePetscMatrix(DOFVector<double> *vec);
virtual ProblemStatBase *getProblem(int number = 0) {}
virtual void serialize(std::ostream&) {}
......@@ -168,7 +170,7 @@ namespace AMDiS {
Vec petscRhsVec;
Vec petscSolutionVec;
Vec petscSolVec;
/// Number of DOFs in the rank mesh.
int nRankDOFs;
......
AMDiS/src/VtkWriter.cc
View file @ be06847b
......@@ -75,8 +75,7 @@ namespace AMDiS {
return 0;
}
void VtkWriter::writeFile(DOFVector<double> *values,
const char *filename)
void VtkWriter::writeFile(DOFVector<double> *values, const std::string filename)
{
DataCollector *dc = new DataCollector(values->getFESpace(), values);
......
AMDiS/src/VtkWriter.h
View file @ be06847b
......@@ -51,11 +51,11 @@ namespace AMDiS {
int writeFile(const std::string name);
/// May be used to simply write ParaView files.
static void writeFile(DOFVector<double> *values,
const char *filename);
static void writeFile(DOFVector<double> *values, const std::string filename);
/// Set a compressing method for file output.
void setCompression(FileCompression c) {
void setCompression(FileCompression c)
{
compress = c;
}
......@@ -95,25 +95,25 @@ namespace AMDiS {
#ifdef HAVE_BOOST
/// Writes a world coordinate to a given file.
inline void writeCoord(boost::iostreams::filtering_ostream &file,
WorldVector<double> coord) {
for (int i = 0; i < Global::getGeo(WORLD); i++) {
WorldVector<double> coord)
{
for (int i = 0; i < Global::getGeo(WORLD); i++)
file << " " << std::scientific << coord[i];
}
for (int i = Global::getGeo(WORLD); i < 3; i++) {
for (int i = Global::getGeo(WORLD); i < 3; i++)
file << " 0.0";
}
file << "\n";
}
#endif
/// Writes a world coordinate to a given file.
inline void writeCoord(std::ofstream &file, WorldVector<double> coord) {
for (int i = 0; i < Global::getGeo(WORLD); i++) {
inline void writeCoord(std::ofstream &file, WorldVector<double> coord)
{
for (int i = 0; i < Global::getGeo(WORLD); i++)
file << " " << std::scientific << coord[i];
}
for (int i = Global::getGeo(WORLD); i < 3; i++) {
for (int i = Global::getGeo(WORLD); i < 3; i++)
file << " 0.0";
}
file << "\n";
}
......
AMDiS/src/VtkWriter.hh
View file @ be06847b
......@@ -97,7 +97,7 @@ namespace AMDiS {
for (it2 = it->begin(); it2 != it->end(); ++it2) {
it2->outputIndex = counter++;
writeCoord(file, it2->coords);
}
}
}
// For the second dim case, write also the interpolation points.
......@@ -107,9 +107,8 @@ namespace AMDiS {
for (pointIt.reset(); !pointIt.end(); ++pointIt) {
std::list< WorldVector<double> >::iterator it2;
for (it2 = pointIt->begin(); it2 != pointIt->end(); ++it2) {
for (it2 = pointIt->begin(); it2 != pointIt->end(); ++it2)
writeCoord(file, *it2);
}
}
}
}
......@@ -147,9 +146,8 @@ namespace AMDiS {
++intPointIt, ++valueIt, ++coordIt) {
if (*intPointIt == -2) {
for (int i = 0; i < static_cast<int>(coordIt->size()); i++) {
for (int i = 0; i < static_cast<int>(coordIt->size()); i++)
file << " " << (fabs(*valueIt) < 1e-40 ? 0.0 : *valueIt) << "\n";
}
}
}
......@@ -163,9 +161,8 @@ namespace AMDiS {
++intPointIt, ++valueIt, ++interpCoordIt) {
if (*intPointIt >= 0) {
for (int i = 0; i < static_cast<int>(interpCoordIt->size()); i++) {
for (int i = 0; i < static_cast<int>(interpCoordIt->size()); i++)
file << " " << (fabs(*valueIt) < 1e-40 ? 0.0 : *valueIt) << "\n";
}
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment