Commit 576bb4eb authored by Thomas Witkowski's avatar Thomas Witkowski

Ups, mal ein code refactoring dass nicht gleich in 100 bugfixes enden. So, nun...

Ups, mal ein code refactoring dass nicht gleich in 100 bugfixes enden. So, nun sollte der ganze parallele spass auch fuer gekoppelte Probleme ohne Tricks funktionieren. Nur testen will es mal wieder keiner, daher geh ich mittag essen.
parent ecb93cc3
......@@ -328,13 +328,13 @@ namespace AMDiS {
}
/// Returns \ref feSpaces.
inline vector<const FiniteElemSpace*> getFeSpaces()
inline vector<const FiniteElemSpace*>& getFeSpaces()
{
return feSpaces;
}
/// Returns \ref componentSpaces;
inline vector<const FiniteElemSpace*> getComponentSpaces()
inline vector<const FiniteElemSpace*>& getComponentSpaces()
{
return componentSpaces;
}
......
......@@ -89,7 +89,6 @@ namespace AMDiS {
MSG("nelem = %d\n", nelem);
// global number of nodes
ParallelDofMapping &dofMap = meshDistributor->getDofMap();
int nnod = dofMap[feSpace].nOverallDofs;
MSG("nnod = %d\n", nnod);
......@@ -443,8 +442,6 @@ namespace AMDiS {
typedef traits::range_generator<row, Matrix>::type cursor_type;
typedef traits::range_generator<nz, cursor_type>::type icursor_type;
ParallelDofMapping &dofMap = meshDistributor->getDofMap();
for (cursor_type cursor = begin<row>(dmat->getBaseMatrix()),
cend = end<row>(dmat->getBaseMatrix()); cursor != cend; ++cursor) {
for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor);
......
This diff is collapsed.
......@@ -63,10 +63,20 @@ namespace AMDiS {
public:
~MeshDistributor();
/// Initialization of mesh distributor.
void initParallelization();
/// Clean up procedure for the mesh distributor and attached objects.
void exitParallelization();
/** \brief
* Register a parallel DOF mapping. This DOF mapping object will than
* automatically updated by the mesh distributer after mesh changes.
*
* \param[in] dofMap Parallel DOF mapping object.
*/
void registerDofMap(ParallelDofMapping &dofMap);
/// Adds a DOFVector to the set of \ref interchangeVecs. Thus, this vector
/// will be automatically interchanged between ranks when mesh is
/// repartitioned.
......@@ -132,49 +142,6 @@ namespace AMDiS {
return mesh;
}
/// Returns an FE space from \ref feSpaces.
inline const FiniteElemSpace* getFeSpace(unsigned int i = 0)
{
FUNCNAME("MeshDistributor::getFeSpace()");
TEST_EXIT_DBG(i < feSpaces.size())
("Try to access FE space %d, but have only %d FE spaces!\n",
i, feSpaces.size());
return feSpaces[i];
}
/// Returns all FE spaces, thus \ref feSpaces.
inline vector<const FiniteElemSpace*>& getFeSpaces()
{
return feSpaces;
}
inline const FiniteElemSpace* getComponentSpace(unsigned int i = 0)
{
FUNCNAME("MeshDistributor::getFeSpace()");
TEST_EXIT_DBG(i < componentSpaces.size())("Should not happen!\n");
return componentSpaces[i];
}
inline vector<const FiniteElemSpace*>& getComponentSpaces()
{
return componentSpaces;
}
/// Returns the DOF mapping object, \ref dofMap.
inline ParallelDofMapping& getDofMap()
{
return dofMap;
}
inline ParallelDofMapping& getDofMapSd()
{
return dofMapSd;
}
/// Returns the periodic mapping handler, \ref periodicMap.
inline PeriodicMap& getPeriodicMap()
{
......@@ -360,7 +327,6 @@ namespace AMDiS {
/// Updates the local and global DOF numbering after the mesh has been
/// changed.
void updateLocalGlobalNumbering(ParallelDofMapping &dmap,
DofComm &dcom,
const FiniteElemSpace *feSpace);
protected:
......@@ -501,14 +467,12 @@ namespace AMDiS {
/// Name of the problem (as used in the init files)
string name;
/// Finite element spaces of the problem.
vector<const FiniteElemSpace*> componentSpaces;
/// Set of all different FE spaces.
vector<const FiniteElemSpace*> feSpaces;
/// Mesh of the problem.
/// Pointer to the only mesh. Note that we do not yet support multi mesh
/// method, thus even if we consider coupled problems, all problems must
/// be defined on the same mesh.
Mesh *mesh;
/// A refinement manager that should be used on the mesh. It is used to
......@@ -516,9 +480,6 @@ namespace AMDiS {
/// elements on the other side of the interior boundary.
RefinementManager *refineManager;
/// Info level.
int info;
/// Pointer to a mesh partitioner that is used to partition the mesh to
/// the ranks.
MeshPartitioner *partitioner;
......@@ -531,11 +492,6 @@ namespace AMDiS {
/// macro element.
map<int, int> partitionMap;
/// Mapping object to map from local DOF indices to global ones.
ParallelDofMapping dofMap;
ParallelDofMapping dofMapSd;
/// Database to store and query all sub-objects of all elements of the
/// macro mesh.
ElementObjectDatabase elObjDb;
......@@ -608,6 +564,8 @@ namespace AMDiS {
/// all boundary DOFs.
vector<map<const FiniteElemSpace*, BoundaryDofInfo> > boundaryDofInfo;
/// Stores information about hierarchical decomposition of the mesh levels.
/// Is used to specify multi level solver methods.
MeshLevelData levelData;
/// If there is no mesh adaptivity, the mesh distributor can remove some
......@@ -616,6 +574,10 @@ namespace AMDiS {
/// is set to true, and thus no special assumption are made.
bool meshAdaptivity;
/// Set of all parallel DOF mapping object that are registered by parallel
/// solver objects and must be updated automatically after mesh change.
vector<ParallelDofMapping*> dofMaps;
public:
static bool sebastianMode;
......
......@@ -115,8 +115,6 @@ namespace AMDiS {
if (checkMeshChange()) {
// Mesh has been changed, recompute interior DOF mapping.
vector<const FiniteElemSpace*> feSpaces =
meshDistributor->getComponentSpaces();
interiorMap->setComputeMatIndex(!localMatrix);
interiorMap->update();
......
......@@ -462,7 +462,7 @@ namespace AMDiS {
mpi::globalAdd(foundError);
TEST_EXIT(foundError == 0)("Error found on at least on rank!\n");
INFO(pdb.info, 8)("Test common dofs needed %.5f seconds\n", TIME_USED(first, clock()));
MSG("Test common dofs needed %.5f seconds\n", TIME_USED(first, clock()));
}
......@@ -481,7 +481,7 @@ namespace AMDiS {
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) {
coordsToIndex[(*it)] = pdb.dofMap[feSpace][it.getDOFIndex()].global;
coordsToIndex[(*it)] = (*(pdb.dofMaps[0]))[feSpace][it.getDOFIndex()].global;
// MSG(" CHECK FOR DOF %d AT COORDS %f %f %f\n",
// coordsToIndex[(*it)], (*it)[0], (*it)[1], (*it)[2]);
}
......
......@@ -243,6 +243,7 @@ namespace AMDiS {
levelData = &ldata;
globalMapping = b;
componentSpaces = fe;
feSpaces = uniqueFe;
data->init(fe, uniqueFe, globalMapping, ldata);
}
......
......@@ -151,7 +151,8 @@ namespace AMDiS {
{
FUNCNAME("ComponentDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)
("DOF %d is already defined in mapping!\n", dof0);
dofMap[dof0].local = dof1;
nLocalDofs++;
......@@ -637,17 +638,17 @@ namespace AMDiS {
* Initialize the parallel DOF mapping.
*
* \param[in] m MPI communicator.
* \param[in] fe The FE spaces of all components of the
* \param[in] componentSpaces The FE spaces of all components of the
* PDE to be solved.
* \param[in] uniqueFe Unique list of FE spaces. Thus, two
* \param[in] feSpaces Unique list of FE spaces. Thus, two
* arbitrary elements of this list are always
* different.
* \param[in] globalMapping If true, at least one rank's mapping con-
* taines DOFs that are not owend by the rank.
*/
void init(MeshLevelData& mld,
vector<const FiniteElemSpace*> &fe,
vector<const FiniteElemSpace*> &uniqueFe,
vector<const FiniteElemSpace*> &componentSpaces,
vector<const FiniteElemSpace*> &feSpaces,
bool globalMapping = true);
/// In the case of having only one FE space, this init function can be used.
......@@ -774,6 +775,12 @@ namespace AMDiS {
return dofToMatIndex.get(ithComponent, d) - rStartDofs;
}
/// Returns the set of unique FE spaces.
inline vector<const FiniteElemSpace*>& getFeSpaces()
{
return feSpaces;
}
// Writes all data of this object to an output stream.
void serialize(ostream &out)
{
......@@ -871,8 +878,11 @@ namespace AMDiS {
/// consideration of possibly multiple components.
DofToMatIndex dofToMatIndex;
/// FE spaces of all components
/// FE spaces of all components.
vector<const FiniteElemSpace*> componentSpaces;
/// Set of unique FE spaces.
vector<const FiniteElemSpace*> feSpaces;
};
}
......
......@@ -70,9 +70,14 @@ namespace AMDiS {
ProblemStatSeq* adoptProblem,
Flag adoptFlag)
{
ParallelProblemStatBase::initialize(initFlag, adoptProblem, adoptFlag);
ParallelProblemStatBase::initialize(initFlag, adoptProblem, adoptFlag);
meshDistributor->setBoundaryDofRequirement(petscSolver->getBoundaryDofRequirement());
meshDistributor->setBoundaryDofRequirement(petscSolver->getBoundaryDofRequirement());
petscSolver->setMeshDistributor(meshDistributor,
meshDistributor->getMpiComm(),
PETSC_COMM_SELF);
petscSolver->init(this->getComponentSpaces(), this->getFeSpaces());
}
......@@ -86,13 +91,8 @@ namespace AMDiS {
double wtime = MPI::Wtime();
if (createMatrixData) {
petscSolver->setMeshDistributor(meshDistributor,
meshDistributor->getMpiComm(),
PETSC_COMM_SELF);
petscSolver->setDofMapping(&(meshDistributor->getDofMap()));
if (createMatrixData)
petscSolver->fillPetscMatrix(systemMatrix);
}
petscSolver->fillPetscRhs(rhs);
......
......@@ -21,13 +21,17 @@ namespace AMDiS {
using namespace std;
PetscSolver::PetscSolver()
: ParallelCoarseSpaceMatVec(),
: ParallelCoarseSpaceMatVec(),
dofMap(FESPACE_WISE),
dofMapSd(FESPACE_WISE),
kspPrefix(""),
removeRhsNullspace(false),
hasConstantNullspace(false),
isSymmetric(false),
handleDirichletRows(true)
{
setDofMapping(&dofMap);
string kspStr = "";
Parameters::get("parallel->solver->petsc->ksp", kspStr);
if (kspStr != "")
......@@ -40,6 +44,38 @@ namespace AMDiS {
}
void PetscSolver::init(vector<const FiniteElemSpace*> &fe0,
vector<const FiniteElemSpace*> &fe1)
{
FUNCNAME("PetscSolver::init()");
TEST_EXIT(meshDistributor)("No mesh distributor object defined!\n");
TEST_EXIT(fe0.size())("No component spaces in PETSc solver object!\n");
TEST_EXIT(fe1.size())("No FE spaces in PETSc solver object!\n");
componentSpaces = fe0;
feSpaces = fe1;
MeshLevelData& levelData = meshDistributor->getMeshLevelData();
int nLevels = levelData.getLevelNumber();
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
dofMap.init(levelData, componentSpaces, feSpaces);
dofMap.setMpiComm(levelData.getMpiComm(0), 0);
dofMap.setDofComm(meshDistributor->getDofComm());
dofMap.clear();
meshDistributor->registerDofMap(dofMap);
if (nLevels > 1) {
dofMapSd.init(levelData, componentSpaces, feSpaces);
dofMapSd.setMpiComm(levelData.getMpiComm(1), 1);
dofMapSd.setDofComm(meshDistributor->getDofCommSd());
dofMapSd.clear();
meshDistributor->registerDofMap(dofMapSd);
}
}
void PetscSolver::fillPetscMatrix(DOFMatrix* mat)
{
Matrix<DOFMatrix*> sysMat(1, 1);
......
......@@ -54,6 +54,9 @@ namespace AMDiS {
virtual ~PetscSolver() {}
virtual void init(vector<const FiniteElemSpace*> &componentSpaces,
vector<const FiniteElemSpace*> &feSpaces);
/** \brief
* Create a PETSc matrix. The given DOF matrices are used to create the nnz
* structure of the PETSc matrix and the values are transfered to it.
......@@ -159,6 +162,16 @@ namespace AMDiS {
/// Run test, if matrix is symmetric.
bool testMatrixSymmetric(Mat mat, bool advancedTest = false);
protected:
/// FE spaces of all components for the stationary problem the specific
/// solver object is registered to.
vector<const FiniteElemSpace*> componentSpaces;
/// Set of unique FE spaces in \ref componentSpaces.
vector<const FiniteElemSpace*> feSpaces;
///
ParallelDofMapping dofMap, dofMapSd;
/// PETSc solver object
KSP kspInterior;
......
This diff is collapsed.
......@@ -69,9 +69,8 @@ namespace AMDiS {
MeshDistributor::BOUNDARY_FILL_INFO_RECV_DOFS;
}
/// Initialization of the data structures with a given list of the FE
/// spaces of all components.
void initialize(vector<const FiniteElemSpace*> feSpaces);
/// Initialization of the FETI-DPdata structures.
void initialize();
int getNumberOfPrimals()
{
......@@ -109,35 +108,29 @@ namespace AMDiS {
/// Defines which boundary nodes are primal. Creates global index of
/// the primal variables.
void createPrimals(int component,
const FiniteElemSpace *feSpace);
void createPrimals(int component);
/// Defines the set of dual variables and creates the global index of
/// dual variables.
void createDuals(int component,
const FiniteElemSpace *feSpace);
void createDuals(int component);
///
void createInterfaceNodes(int component,
const FiniteElemSpace *feSpace);
void createInterfaceNodes(int component);
/// Create Lagrange multiplier variables corresponding to the dual
/// variables.
void createLagrange(int component,
const FiniteElemSpace *feSpace);
void createLagrange(int component);
void createAugmentedLagrange(int component,
const FiniteElemSpace *feSpace);
void createAugmentedLagrange(int component);
/// Creates a global index of the B variables.
void createIndexB(int component,
const FiniteElemSpace *feSpace);
void createIndexB(int component);
/// Creates the Lagrange multiplier constraints and assembles them
/// to \ref mat_lagrange.
void createMatLagrange(vector<const FiniteElemSpace*> &feSpaces);
void createMatLagrange();
void createMatAugmentedLagrange(vector<const FiniteElemSpace*> &feSpaces);
void createMatAugmentedLagrange();
bool testWirebasketEdge(BoundaryObject &edge, const FiniteElemSpace *feSpace);
......@@ -146,7 +139,7 @@ namespace AMDiS {
/// Creates PETSc KSP solver object for solving the Schur complement
/// system on the primal variables, \ref ksp_schur_primal
void createSchurPrimalKsp(vector<const FiniteElemSpace*> &feSpaces);
void createSchurPrimalKsp();
///
void createMatExplicitSchurPrimal();
......@@ -158,7 +151,7 @@ namespace AMDiS {
void destroySchurPrimalKsp();
/// Creates PETSc KSP solver object for the FETI-DP operator, \ref ksp_feti
void createFetiKsp(vector<const FiniteElemSpace*> &feSpaces);
void createFetiKsp();
/// Destroys FETI-DP operator, \ref ksp_feti
void destroyFetiKsp();
......
......@@ -74,10 +74,10 @@ namespace AMDiS {
localDofMap.createVec(ktest2, feti.nGlobalOverallInterior);
const FiniteElemSpace* pressureFeSpace =
feti.meshDistributor->getComponentSpace(feti.pressureComponent);
feti.componentSpaces[feti.pressureComponent];
DofMap& m = localDofMap[feti.pressureComponent].getMap();
for (DofMap::iterator it = m.begin(); it != m.end(); ++it) {
if (feti.meshDistributor->getDofMap()[pressureFeSpace].isRankDof(it->first)) {
if (feti.dofMap[pressureFeSpace].isRankDof(it->first)) {
int index = localDofMap.getLocalMatIndex(feti.pressureComponent, it->first);
VecSetValue(ktest0, index, 1.0, INSERT_VALUES);
}
......
......@@ -28,7 +28,7 @@ namespace AMDiS {
createMatVec(*seqMat);
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
nComponents = seqMat->getNumRows();
int nRankRows = (*interiorMap)[feSpace].nRankDofs;
int nOverallRows = (*interiorMap)[feSpace].nOverallDofs;
......@@ -109,7 +109,7 @@ namespace AMDiS {
TEST_EXIT_DBG(vec)("NO DOF vector defined!\n");
nComponents = vec->getSize();
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
int nRankRows = (*interiorMap)[feSpace].nRankDofs;
int nOverallRows = (*interiorMap)[feSpace].nOverallDofs;
......@@ -139,7 +139,7 @@ namespace AMDiS {
KSPGetPC(kspInterior, &pcInterior);
setBlockPreconditioner(pcInterior);
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
VecDuplicate(getVecRhsInterior(), &petscSolVec);
// PETSc.
......@@ -205,7 +205,7 @@ namespace AMDiS {
TEST_EXIT(petscMat)("No PETSc matrix!\n");
TEST_EXIT(seqMat)("No DOFMatrix!\n");
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits = mtl::traits;
......@@ -262,7 +262,7 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverGlobalBlockMatrix::setDofVector()");
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
// Traverse all used DOFs in the dof vector.
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
......
......@@ -82,7 +82,7 @@ namespace AMDiS {
if (!zeroStartVector)
KSPSetInitialGuessNonzero(kspInterior, PETSC_TRUE);
removeDirichletRows(seqMat, meshDistributor->getDofMap(), getMatInterior());
removeDirichletRows(seqMat, dofMap, getMatInterior());
#if (DEBUG != 0)
MSG("Fill petsc matrix 3 needed %.5f seconds\n", MPI::Wtime() - wtime);
......@@ -111,8 +111,6 @@ namespace AMDiS {
TEST_EXIT_DBG(coarseSpaceMap.size() == seqMat->getSize())
("Wrong sizes %d %d\n", coarseSpaceMap.size(), seqMat->getSize());
vector<const FiniteElemSpace*> feSpaces = meshDistributor->getComponentSpaces();
// === Prepare traverse of sequentially created matrices. ===
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
......@@ -277,7 +275,7 @@ namespace AMDiS {
vecRhsAssembly();
removeDirichletRows(vec, meshDistributor->getDofMap(), getVecRhsInterior());
removeDirichletRows(vec, dofMap, getVecRhsInterior());
// === For debugging allow to write the rhs vector to a file. ===
......
......@@ -25,7 +25,7 @@ namespace AMDiS {
TEST_EXIT_DBG(meshDistributor)("No mesh distributor object defined!\n");
TEST_EXIT_DBG(interiorMap)("No parallel DOF map defined!\n");
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
typedef map<int, DofContainer> RankToDofContainer;
typedef map<DegreeOfFreedom, bool> DofIndexToBool;
......@@ -182,7 +182,7 @@ namespace AMDiS {
createMatVec(*seqMat);
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
int nComponents = seqMat->getNumRows();
updateDofData(nComponents);
......@@ -257,7 +257,7 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverSchur::fillPetscRhs()");
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
int nComponents = vec->getSize();
int nRankRows = (*interiorMap)[feSpace].nRankDofs * nComponents;
int nOverallRows = (*interiorMap)[feSpace].nOverallDofs * nComponents;
......@@ -276,7 +276,7 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverSchur::solvePetscMatrix()");
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
int nComponents = vec.getSize();
KSPCreate(mpiCommGlobal, &kspInterior);
......@@ -345,7 +345,7 @@ namespace AMDiS {
TEST_EXIT(seqMat)("No DOFMatrix!\n");
const FiniteElemSpace* feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace* feSpace = componentSpaces[0];
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits= mtl::traits;
typedef DOFMatrix::base_matrix_type Matrix;
......@@ -431,7 +431,7 @@ namespace AMDiS {
void PetscSolverSchur::setDofVector(Vec& petscVec, DOFVector<double>* vec,
int dispMult, int dispAdd, bool rankOnly)
{
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
const FiniteElemSpace *feSpace = componentSpaces[0];
DOFVector<double>::Iterator dofIt(vec, USED_DOFS);
for (dofIt.reset(); !dofIt.end(); ++dofIt) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment