Commit dfd9bc02 authored by Siqi Ling's avatar Siqi Ling
Browse files

merge parallel multimesh branch to the trunk

parent 4584272c
...@@ -164,6 +164,10 @@ namespace AMDiS { ...@@ -164,6 +164,10 @@ namespace AMDiS {
template<typename T> class WorldMatrix; template<typename T> class WorldMatrix;
template<typename T> class VectorOfFixVecs; template<typename T> class VectorOfFixVecs;
namespace detail {
template <class P> class CouplingProblemStat;
}
typedef mtl::dense2D<double> ElementMatrix; typedef mtl::dense2D<double> ElementMatrix;
typedef mtl::dense_vector<double> ElementVector; typedef mtl::dense_vector<double> ElementVector;
......
...@@ -53,7 +53,6 @@ namespace AMDiS { ...@@ -53,7 +53,6 @@ namespace AMDiS {
Assembler::~Assembler() Assembler::~Assembler()
{} {}
void Assembler::calculateElementMatrix(const ElInfo *elInfo, void Assembler::calculateElementMatrix(const ElInfo *elInfo,
ElementMatrix& userMat, ElementMatrix& userMat,
double factor) double factor)
...@@ -64,14 +63,17 @@ namespace AMDiS { ...@@ -64,14 +63,17 @@ namespace AMDiS {
Element *el = elInfo->getElement(); Element *el = elInfo->getElement();
if (el != lastMatEl || !operat->isOptimized()) { if (el != lastMatEl || !operat->isOptimized()) {
initElement(elInfo, elInfo); initElement(elInfo);
if (rememberElMat) if (rememberElMat)
set_to_zero(elementMatrix); set_to_zero(elementMatrix);
lastMatEl = el; lastMatEl = el;
} else { } else {
// Only possible in single mesh case when one operator
// is used more than twice?
if (rememberElMat) { if (rememberElMat) {
if (&userMat != &elementMatrix)
userMat += factor * elementMatrix; userMat += factor * elementMatrix;
return; return;
} }
...@@ -92,7 +94,6 @@ namespace AMDiS { ...@@ -92,7 +94,6 @@ namespace AMDiS {
userMat += factor * elementMatrix; userMat += factor * elementMatrix;
} }
void Assembler::calculateElementMatrix(const ElInfo *rowElInfo, void Assembler::calculateElementMatrix(const ElInfo *rowElInfo,
const ElInfo *colElInfo, const ElInfo *colElInfo,
const ElInfo *smallElInfo, const ElInfo *smallElInfo,
...@@ -101,108 +102,61 @@ namespace AMDiS { ...@@ -101,108 +102,61 @@ namespace AMDiS {
ElementMatrix& userMat, ElementMatrix& userMat,
double factor) double factor)
{ {
if (remember && (factor != 1.0 || operat->uhOld))
rememberElMat = true;
Element *el = smallElInfo->getElement(); Element *el = smallElInfo->getElement();
// initElement and calculateElementMatrix always need to be
// recalculated since the smaller ElInfo can be different
lastVecEl = lastMatEl = NULL; lastVecEl = lastMatEl = NULL;
if ((el != lastMatEl && el != lastVecEl) || !operat->isOptimized()) if ((el != lastMatEl && el != lastVecEl) || !operat->isOptimized())
initElement(smallElInfo, largeElInfo); initElement(smallElInfo, largeElInfo);
if (el != lastMatEl || !operat->isOptimized()) {
if (rememberElMat)
set_to_zero(elementMatrix);
if (el != lastMatEl || !operat->isOptimized())
lastMatEl = el; lastMatEl = el;
} else { else {
if (rememberElMat) { ERROR_EXIT("Never reached because lastVecEl = lastMatEl = NULL;\n");
if (&userMat != &elementMatrix)
userMat += factor * elementMatrix; userMat += factor * elementMatrix;
return; return;
} }
}
ElementMatrix& mat = rememberElMat ? elementMatrix : userMat; // In multimesh case, elementMatrix is always be recalculated, because each time we
// need to do transformation here.
if (secondOrderAssembler) { set_to_zero(elementMatrix);
// calculate element matrices always on smallest element
secondOrderAssembler->calculateElementMatrix(smallElInfo, mat);
// smallElInfo stores refinement-relation to largeElInfo
ElementMatrix &m =
smallElInfo->getSubElemGradCoordsMat(rowFeSpace->getBasisFcts()->getDegree()); // muste be moved to next if-else block when generalized for multiple polynomial degrees
if (!rowColFeSpaceEqual) { ElementMatrix& mat = elementMatrix;
if (smallElInfo == colElInfo)
tmpMat = m * mat;
else
tmpMat = mat * trans(m);
mat = tmpMat; if (secondOrderAssembler)
} secondOrderAssembler->calculateElementMatrix(smallElInfo, mat);
} if (firstOrderAssemblerGrdPsi)
if (firstOrderAssemblerGrdPsi) {
firstOrderAssemblerGrdPsi->calculateElementMatrix(smallElInfo, mat); firstOrderAssemblerGrdPsi->calculateElementMatrix(smallElInfo, mat);
if (firstOrderAssemblerGrdPhi)
if (!rowColFeSpaceEqual) {
if (largeElInfo == rowElInfo) {
ElementMatrix &m =
smallElInfo->getSubElemGradCoordsMat(rowFeSpace->getBasisFcts()->getDegree());
tmpMat = m * mat;
} else {
ElementMatrix &m =
smallElInfo->getSubElemCoordsMat(rowFeSpace->getBasisFcts()->getDegree());
tmpMat = mat * trans(m);
}
mat = tmpMat;
}
}
if (firstOrderAssemblerGrdPhi) {
firstOrderAssemblerGrdPhi->calculateElementMatrix(smallElInfo, mat); firstOrderAssemblerGrdPhi->calculateElementMatrix(smallElInfo, mat);
if (zeroOrderAssembler)
zeroOrderAssembler->calculateElementMatrix(smallElInfo, mat);
if (!rowColFeSpaceEqual) {
if (largeElInfo == colElInfo) {
ElementMatrix &m =
smallElInfo->getSubElemGradCoordsMat(rowFeSpace->getBasisFcts()->getDegree());
tmpMat = mat * trans(m);
} else {
ElementMatrix &m =
smallElInfo->getSubElemCoordsMat(rowFeSpace->getBasisFcts()->getDegree());
tmpMat = m * mat;
}
mat = tmpMat;
}
}
if (zeroOrderAssembler) {
zeroOrderAssembler->calculateElementMatrix(smallElInfo, mat);
if (!rowColFeSpaceEqual) {
ElementMatrix &m = ElementMatrix &m =
smallElInfo->getSubElemCoordsMat(rowFeSpace->getBasisFcts()->getDegree()); smallElInfo->getSubElemCoordsMat(rowFeSpace->getBasisFcts()->getDegree());
if (!rowColFeSpaceEqual) {
if (smallElInfo == colElInfo) if (smallElInfo == colElInfo)
tmpMat = m * mat; tmpMat = m * mat;
else else
tmpMat = mat * trans(m); tmpMat = mat * trans(m);
mat = tmpMat; mat = tmpMat;
} } else if (smallElInfo == colElInfo) {
tmpMat = m * mat * trans(m);
mat = tmpMat;
} }
if (rememberElMat && &userMat != &elementMatrix) // This is for the call in matVecAssemble
if (&userMat != &elementMatrix)
userMat += factor * elementMatrix; userMat += factor * elementMatrix;
} }
void Assembler::calculateElementVector(const ElInfo *elInfo, void Assembler::calculateElementVector(const ElInfo *elInfo,
ElementVector& userVec, ElementVector& userVec,
double factor) double factor)
...@@ -221,6 +175,8 @@ namespace AMDiS { ...@@ -221,6 +175,8 @@ namespace AMDiS {
lastVecEl = el; lastVecEl = el;
} else { } else {
// Only possible in single mesh case when one operator
// is used more than twice at dof vector?
if (rememberElVec) { if (rememberElVec) {
userVec += factor * elementVector; userVec += factor * elementVector;
return; return;
...@@ -246,7 +202,6 @@ namespace AMDiS { ...@@ -246,7 +202,6 @@ namespace AMDiS {
userVec += factor * elementVector; userVec += factor * elementVector;
} }
void Assembler::calculateElementVector(const ElInfo *mainElInfo, void Assembler::calculateElementVector(const ElInfo *mainElInfo,
const ElInfo *auxElInfo, const ElInfo *auxElInfo,
const ElInfo *smallElInfo, const ElInfo *smallElInfo,
...@@ -256,36 +211,33 @@ namespace AMDiS { ...@@ -256,36 +211,33 @@ namespace AMDiS {
{ {
FUNCNAME("Assembler::calculateElementVector()"); FUNCNAME("Assembler::calculateElementVector()");
if (remember && factor != 1.0)
rememberElVec = true;
Element *el = mainElInfo->getElement(); Element *el = mainElInfo->getElement();
// initElement and calculateElementVector always need to be
// recalculated since the smaller ElInfo can be different
lastVecEl = lastMatEl = NULL;
if ((el != lastMatEl && el != lastVecEl) || !operat->isOptimized()) if ((el != lastMatEl && el != lastVecEl) || !operat->isOptimized())
initElement(smallElInfo, largeElInfo); initElement(smallElInfo, largeElInfo);
if (el != lastVecEl || !operat->isOptimized()) { if (el != lastVecEl || !operat->isOptimized())
if (rememberElVec)
set_to_zero(elementVector);
lastVecEl = el; lastVecEl = el;
} else { else {
if (rememberElVec) { ERROR_EXIT("Never reached because lastVecEl = lastMatEl = NULL;\n");
userVec += factor * elementVector; userVec += factor * elementVector;
return; return;
} }
}
ElementVector& vec = rememberElVec ? elementVector : userVec; set_to_zero(elementVector);
ElementVector& vec = elementVector;
if (operat->uhOld && remember) { if (operat->uhOld && remember) {
if (smallElInfo->getLevel() == largeElInfo->getLevel()) if (smallElInfo->getLevel() == largeElInfo->getLevel())
matVecAssemble(auxElInfo, vec); matVecAssemble(auxElInfo, vec);
else else
matVecAssemble(mainElInfo, auxElInfo, smallElInfo, largeElInfo, vec); matVecAssemble(mainElInfo, auxElInfo, smallElInfo, largeElInfo, vec);
if (rememberElVec)
userVec += factor * elementVector; userVec += factor * elementVector;
return; return;
} }
...@@ -306,13 +258,12 @@ namespace AMDiS { ...@@ -306,13 +258,12 @@ namespace AMDiS {
} }
} }
if (rememberElVec)
userVec += factor * elementVector; userVec += factor * elementVector;
} }
void Assembler::matVecAssemble(const ElInfo *elInfo, ElementVector& vec) void Assembler::matVecAssemble(const ElInfo *elInfo, ElementVector& vec)
{ {
Element *el = elInfo->getElement(); Element *el = elInfo->getElement();
ElementVector uhOldLoc(operat->uhOld->getFeSpace() == rowFeSpace ? ElementVector uhOldLoc(operat->uhOld->getFeSpace() == rowFeSpace ?
nRow : nCol); nRow : nCol);
...@@ -323,7 +274,8 @@ namespace AMDiS { ...@@ -323,7 +274,8 @@ namespace AMDiS {
calculateElementMatrix(elInfo, elementMatrix); calculateElementMatrix(elInfo, elementMatrix);
} }
vec += elementMatrix*uhOldLoc;
vec += elementMatrix * uhOldLoc;
} }
...@@ -357,6 +309,7 @@ namespace AMDiS { ...@@ -357,6 +309,7 @@ namespace AMDiS {
} }
vec += elementMatrix * uhOldLoc; vec += elementMatrix * uhOldLoc;
} }
......
...@@ -57,6 +57,8 @@ namespace AMDiS { ...@@ -57,6 +57,8 @@ namespace AMDiS {
bool reverseMode = false; bool reverseMode = false;
TEST_EXIT_DBG(obj0.el && obj1.el)("BoundaryObject without element pointer.\n");
switch (feSpace->getMesh()->getDim()) { switch (feSpace->getMesh()->getDim()) {
case 2: case 2:
reverseMode = true; reverseMode = true;
......
...@@ -99,7 +99,7 @@ namespace AMDiS { ...@@ -99,7 +99,7 @@ namespace AMDiS {
#if HAVE_PARALLEL_DOMAIN_AMDIS #if HAVE_PARALLEL_DOMAIN_AMDIS
vector<FixRefinementPatch::EdgeInEl> refineEdges; vector<FixRefinementPatch::EdgeInEl> refineEdges;
FixRefinementPatch::getOtherEl(stack, refineEdges); FixRefinementPatch::getOtherEl(mesh, stack, refineEdges);
// === If the refinement edge must be fixed, add also the other part of this === // === If the refinement edge must be fixed, add also the other part of this ===
// === edge to the refinement patch. === // === edge to the refinement patch. ===
...@@ -110,6 +110,7 @@ namespace AMDiS { ...@@ -110,6 +110,7 @@ namespace AMDiS {
return; return;
Element *otherEl = refineEdges[0].first; Element *otherEl = refineEdges[0].first;
TEST_EXIT_DBG(otherEl->getMesh() == mesh)("Something is wrong.\n");
TraverseStack stack2; TraverseStack stack2;
ElInfo *elInfo2 = ElInfo *elInfo2 =
......
...@@ -40,7 +40,7 @@ namespace AMDiS { ...@@ -40,7 +40,7 @@ namespace AMDiS {
* problem-list, that is filled by addProblem. Alternatively one can a access * problem-list, that is filled by addProblem. Alternatively one can a access
* each problem by an unique name. * each problem by an unique name.
*/ */
class CouplingIterationInterface : public ProblemIterationInterface class CouplingIterationInterface : public virtual ProblemIterationInterface
{ {
public: public:
virtual ~CouplingIterationInterface() {} virtual ~CouplingIterationInterface() {}
......
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
#include "AMDiS_fwd.h" #include "AMDiS_fwd.h"
#include "ProblemStat.h" #include "ProblemStat.h"
#include "Initfile.h" #include "Initfile.h"
#include <boost/lexical_cast.hpp> #include "utility/to_string.hpp"
namespace AMDiS { namespace AMDiS {
...@@ -43,21 +43,26 @@ namespace AMDiS { ...@@ -43,21 +43,26 @@ namespace AMDiS {
* computations. * computations.
*/ */
template<typename ProblemStatType> template<typename ProblemStatType>
class CouplingProblemStat class CouplingProblemStat : public ProblemStatSeq
{ {
protected:
typedef ProblemStatSeq super;
using super::nComponents;
using super::meshes;
using super::nMeshes;
using super::feSpaces;
using super::name;
using super::refinementManager;
using super::coarseningManager;
public: public:
/// Constructor /// Constructor
CouplingProblemStat(std::string name_) CouplingProblemStat(std::string name_)
: name(name_), : super(name_),
nComponents(0), dim(-1)
nMeshes(0),
refinementManager(NULL),
coarseningManager(NULL)
{} {}
/// Destructor
virtual ~CouplingProblemStat() {}
/// add problem by number /// add problem by number
virtual void addProblem(ProblemStatType* prob) virtual void addProblem(ProblemStatType* prob)
{ {
...@@ -71,112 +76,193 @@ namespace AMDiS { ...@@ -71,112 +76,193 @@ namespace AMDiS {
Flag adoptFlag = INIT_NOTHING) Flag adoptFlag = INIT_NOTHING)
{ FUNCNAME("CouplingProblemStat::initialize()"); { FUNCNAME("CouplingProblemStat::initialize()");
// create one refinement-/coarseningmanager for all problems super::initialize(initFlag - INIT_MESH);
if (refinementManager != NULL && coarseningManager != NULL) {
WARNING("refinement-/coarseningmanager already created\n"); const Flag DEFAULT_INIT = (INIT_FE_SPACE | INIT_MESH | CREATE_MESH | INIT_SYSTEM | INIT_SOLVER | INIT_ESTIMATOR | INIT_MARKER | INIT_FILEWRITER);
} else { for (size_t p = 0; p < problems.size(); ++p) {
if (!adoptProblem) problems[p]->initialize(initFlag - DEFAULT_INIT);
createRefCoarseManager();
else {
refinementManager = adoptProblem->getRefinementManager();
coarseningManager = adoptProblem->getCoarseningManager();
}
} }
if (refinementManager == NULL || coarseningManager == NULL) for (size_t i = 0; i < meshes.size(); i++) {
WARNING("no refinement-/coarseningmanager created\n"); int globalRefinements = 0;
// create Meshes and FeSpaces // If AMDiS is compiled for parallel computations, the global refinements are
// ignored here. Later, each rank will add the global refinements to its
// private mesh.
#ifndef HAVE_PARALLEL_DOMAIN_AMDIS
Parameters::get(meshes[i]->getName() + "->global refinements",
globalRefinements);
#endif
bool initMesh = initFlag.isSet(INIT_MESH);
// Initialize the meshes if there is no serialization file.
if (initMesh && meshes[i] && !(meshes[i]->isInitialized())) {
meshes[i]->initialize();
refinementManager->globalRefine(meshes[i], globalRefinements);
}
}
}
/// Used in \ref initialize().
virtual void createMesh() override
{
// all problems must have the same dimension (?) // all problems must have the same dimension (?)
int dim = 0; dim = 0;
Parameters::get(name + "->dim", dim); Parameters::get(name + "->dim", dim);
TEST_EXIT(dim)("No problem dimension specified for \"%s->dim\"!\n", TEST_EXIT(dim)("No problem dimension specified for \"%s->dim\"!\n",
name.c_str()); name.c_str());
std::map<std::string, Mesh*> meshByName; std::map<std::pair<std::string, int>, Mesh*> meshByName; // (name, refSet) --> Mesh*
typedef std::map<std::pair<std::string, int>, Mesh*>::iterator MeshIterator;
std::vector<std::set<Mesh*> > meshesForProblems(problems.size()); std::vector<std::set<Mesh*> > meshesForProblems(problems.size());
std::map<std::pair<Mesh*, int>, FiniteElemSpace*> feSpaceMap;
for (size_t i = 0; i < problems.size(); ++i) { for (size_t i = 0; i < problems.size(); ++i) {
TEST_EXIT(problems[i])("problem[%d] does not exist!\n",i); TEST_EXIT(problems[i])("problem[%d] does not exist!\n",i);
for (size_t j = 0; j < problems[i]->getNumComponents(); j++) {
// mesh
int nComponents = problems[i]->getNumComponents();
int nAddComponents = 0;
Parameters::get(problems[i]->getName() + "->additional components", nAddComponents);
problems[i]->componentMeshes.resize(nComponents + nAddComponents);
for (size_t j = 0; j < nComponents + nAddComponents; j++) {
// name of the mesh
std::string meshName(""); std::string meshName("");
Parameters::get(problems[i]->getName() + "->mesh", meshName); Parameters::get(problems[i]->getName() + "->mesh", meshName);
TEST_EXIT(meshName != "")("No mesh name specified for \"%s->mesh\"!\n", TEST_EXIT(meshName != "")("No mesh name specified for \"%s->mesh\"!\n",
problems[i]->getName().c_str()); problems[i]->getName().c_str());
if (meshByName.find(meshName) == meshByName.end()) { // dimension of the mesh
int mesh_dim = 0;
Parameters::get(problems[i]->getName() + "->dim", mesh_dim);
TEST_EXIT(dim == mesh_dim)("Mesh-dimension must be the same for all problems!\n");
// refinement set (optional)
int refSet = 0;
Parameters::get(problems[i]->getName() + "->refinement set[" + to_string(j) + "]", refSet);
// create a new Mesh only if not already created for other problem
Mesh* componentMesh;
MeshIterator meshIt = meshByName.find(std::make_pair(meshName, refSet));
if (meshIt == meshByName.end()) {
Mesh *newMesh = new Mesh(meshName, dim); Mesh *newMesh = new Mesh(meshName, dim);
meshByName[meshName] = newMesh;
meshes.push_back(newMesh); meshes.push_back(newMesh);
meshesForProblems[i].insert(newMesh); meshByName[std::make_pair(meshName, refSet)] = newMesh;
componentMesh = newMesh;
nMeshes++; nMeshes++;
} else } else {
meshesForProblems[i].insert(meshByName[meshName]); componentMesh = meshIt->second;
}