Commit 7b1208e2 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on FETI-DP for Stokes like problems.

parent 7b9cb31c
...@@ -61,12 +61,12 @@ namespace AMDiS { ...@@ -61,12 +61,12 @@ namespace AMDiS {
int *verticesEl1 = new int[dim]; int *verticesEl1 = new int[dim];
int *verticesEl2 = new int[dim]; int *verticesEl2 = new int[dim];
int mode = -1; // 0: drop dofs, 1: associate dofs int mode = -1; // 0: drop dofs, 1: associate dofs
int result; int result = 0;
BoundaryType boundaryType; BoundaryType boundaryType;
MacroReader::PeriodicMap periodicMap; MacroReader::PeriodicMap periodicMap;
fscanf(file, "%*s %d", &n); result = fscanf(file, "%*s %d", &n);
fscanf(file, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s"); result = fscanf(file, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s");
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
std::map<int, int> vertexMapEl1; std::map<int, int> vertexMapEl1;
......
...@@ -1692,13 +1692,13 @@ namespace AMDiS { ...@@ -1692,13 +1692,13 @@ namespace AMDiS {
int nLevels = levelData.getLevelNumber(); int nLevels = levelData.getLevelNumber();
TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n"); TEST_EXIT_DBG(nLevels >= 1)("Should not happen!\n");
dofMap.init(levelData, feSpaces, feSpaces, true, true); dofMap.init(levelData, feSpaces, feSpaces);
dofMap.setMpiComm(levelData.getMpiComm(0), 0); dofMap.setMpiComm(levelData.getMpiComm(0), 0);
dofMap.setDofComm(dofComm); dofMap.setDofComm(dofComm);
dofMap.clear(); dofMap.clear();
if (nLevels > 1) { if (nLevels > 1) {
dofMapSd.init(levelData, feSpaces, feSpaces, true, true); dofMapSd.init(levelData, feSpaces, feSpaces);
dofMapSd.setMpiComm(levelData.getMpiComm(1), 1); dofMapSd.setMpiComm(levelData.getMpiComm(1), 1);
dofMapSd.setDofComm(dofCommSd); dofMapSd.setDofComm(dofCommSd);
dofMapSd.clear(); dofMapSd.clear();
......
...@@ -71,7 +71,7 @@ namespace AMDiS { ...@@ -71,7 +71,7 @@ namespace AMDiS {
if (needGlobalMapping) { if (needGlobalMapping) {
computeGlobalMapping(); computeGlobalMapping();
if (hasNonLocalDofs) if (isNonLocal)
computeNonLocalIndices(); computeNonLocalIndices();
} }
} }
...@@ -161,15 +161,14 @@ namespace AMDiS { ...@@ -161,15 +161,14 @@ namespace AMDiS {
void ParallelDofMapping::init(MeshLevelData &ldata, void ParallelDofMapping::init(MeshLevelData &ldata,
vector<const FiniteElemSpace*> &fe, vector<const FiniteElemSpace*> &fe,
vector<const FiniteElemSpace*> &uniqueFe, vector<const FiniteElemSpace*> &uniqueFe,
bool needGlobalMapping, bool b)
bool bNonLocalDofs)
{ {
FUNCNAME("ParallelDofMapping::init()"); FUNCNAME("ParallelDofMapping::init()");
levelData = &ldata; levelData = &ldata;
feSpaces = fe; feSpaces = fe;
feSpacesUnique = uniqueFe; feSpacesUnique = uniqueFe;
hasNonLocalDofs = bNonLocalDofs; isNonLocal = b;
// === Init the mapping for all different FE spaces. === // === Init the mapping for all different FE spaces. ===
...@@ -177,8 +176,8 @@ namespace AMDiS { ...@@ -177,8 +176,8 @@ namespace AMDiS {
for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin(); for (vector<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it) { it != feSpacesUnique.end(); ++it) {
addFeSpace(*it); addFeSpace(*it);
data[*it].setNeedGlobalMapping(needGlobalMapping); data[*it].setNeedGlobalMapping(isNonLocal);
data[*it].setNonLocalDofs(hasNonLocalDofs); data[*it].setNonLocal(isNonLocal);
} }
} }
...@@ -310,8 +309,7 @@ namespace AMDiS { ...@@ -310,8 +309,7 @@ namespace AMDiS {
rStartDofs = computeStartDofs(); rStartDofs = computeStartDofs();
// And finally, compute the matrix indices. // And finally, compute the matrix indices.
if (needMatIndex) computeMatIndex(needMatIndexFromGlobal);
computeMatIndex(needMatIndexFromGlobal);
} }
...@@ -363,7 +361,7 @@ namespace AMDiS { ...@@ -363,7 +361,7 @@ namespace AMDiS {
offset += data[feSpaces[i]].nRankDofs; offset += data[feSpaces[i]].nRankDofs;
// If there are no non local DOFs, continue with the next FE space. // If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs) if (!isNonLocal)
continue; continue;
TEST_EXIT_DBG(dofComm != NULL)("No communicator given!\n"); TEST_EXIT_DBG(dofComm != NULL)("No communicator given!\n");
......
...@@ -115,7 +115,7 @@ namespace AMDiS { ...@@ -115,7 +115,7 @@ namespace AMDiS {
dofComm(NULL), dofComm(NULL),
feSpace(NULL), feSpace(NULL),
needGlobalMapping(false), needGlobalMapping(false),
hasNonLocalDofs(false) isNonLocal(false)
{ {
clear(); clear();
} }
...@@ -230,9 +230,9 @@ namespace AMDiS { ...@@ -230,9 +230,9 @@ namespace AMDiS {
/// Informs the mapping whether the mapping will include DOFs that are not /// Informs the mapping whether the mapping will include DOFs that are not
/// owned by the rank. /// owned by the rank.
void setNonLocalDofs(bool b) void setNonLocal(bool b)
{ {
hasNonLocalDofs = b; isNonLocal = b;
} }
/// Informs the mapping whether a global index must be computed. /// Informs the mapping whether a global index must be computed.
...@@ -287,7 +287,7 @@ namespace AMDiS { ...@@ -287,7 +287,7 @@ namespace AMDiS {
/// Is true if there are DOFs in at least one subdomain that are not owned /// Is true if there are DOFs in at least one subdomain that are not owned
/// by the rank. If the value is false, each rank contains only DOFs that /// by the rank. If the value is false, each rank contains only DOFs that
/// are also owned by this rank. /// are also owned by this rank.
bool hasNonLocalDofs; bool isNonLocal;
public: public:
/// ///
...@@ -306,8 +306,7 @@ namespace AMDiS { ...@@ -306,8 +306,7 @@ namespace AMDiS {
ParallelDofMapping() ParallelDofMapping()
: levelData(NULL), : levelData(NULL),
dofComm(NULL), dofComm(NULL),
hasNonLocalDofs(false), isNonLocal(true),
needMatIndex(false),
needMatIndexFromGlobal(false), needMatIndexFromGlobal(false),
feSpaces(0), feSpaces(0),
nRankDofs(1), nRankDofs(1),
...@@ -329,16 +328,24 @@ namespace AMDiS { ...@@ -329,16 +328,24 @@ namespace AMDiS {
* \param[in] uniqueFe Unique list of FE spaces. Thus, two * \param[in] uniqueFe Unique list of FE spaces. Thus, two
* arbitrary elements of this list are always * arbitrary elements of this list are always
* different. * different.
* \param[in] needGlobalMapping If true, the mapping computes also a global * \param[in] isNonLocal If true, at least one rank's mapping con-
* index for the DOFs.
* \param[in] bNonLocalDofs If true, at least one rank's mapping con-
* taines DOFs that are not owend by the rank. * taines DOFs that are not owend by the rank.
*/ */
void init(MeshLevelData& mld, void init(MeshLevelData& mld,
vector<const FiniteElemSpace*> &fe, vector<const FiniteElemSpace*> &fe,
vector<const FiniteElemSpace*> &uniqueFe, vector<const FiniteElemSpace*> &uniqueFe,
bool needGlobalMapping, bool isNonLocal = true);
bool bNonLocalDofs);
/// In the case of having only one FE space, this init function can be used.
void init(MeshLevelData& mld,
const FiniteElemSpace *feSpace,
bool isNonLocal = true)
{
vector<const FiniteElemSpace*> feSpaces;
feSpaces.push_back(feSpace);
init(mld, feSpaces, feSpaces, isNonLocal);
}
void setMpiComm(MPI::Intracomm &m, int l); void setMpiComm(MPI::Intracomm &m, int l);
...@@ -359,9 +366,10 @@ namespace AMDiS { ...@@ -359,9 +366,10 @@ namespace AMDiS {
return *dofComm; return *dofComm;
} }
void setComputeMatIndex(bool b, bool global = false) /// Changes the computation of matrix indices based of either local or
/// global DOF indices, see \ref needMatIndexFromGlobal
void setComputeMatIndex(bool global)
{ {
needMatIndex = b;
needMatIndexFromGlobal = global; needMatIndexFromGlobal = global;
} }
...@@ -494,11 +502,7 @@ namespace AMDiS { ...@@ -494,11 +502,7 @@ namespace AMDiS {
/// Is true if there are DOFs in at least one subdomain that are not owned /// Is true if there are DOFs in at least one subdomain that are not owned
/// by the rank. If the value is false, each rank contains only DOFs that /// by the rank. If the value is false, each rank contains only DOFs that
/// are also owned by this rank. /// are also owned by this rank.
bool hasNonLocalDofs; bool isNonLocal;
/// If true, matrix indeces for the stored DOFs are computed, see
/// \ref computeMatIndex.
bool needMatIndex;
/// If matrix indices should be computed, this variable defines if the /// If matrix indices should be computed, this variable defines if the
/// mapping from DOF indices to matrix row indices is defined on local /// mapping from DOF indices to matrix row indices is defined on local
......
...@@ -263,6 +263,7 @@ namespace AMDiS { ...@@ -263,6 +263,7 @@ namespace AMDiS {
("Mesh hierarchy does not contain %d levels!\n", meshLevel + 1); ("Mesh hierarchy does not contain %d levels!\n", meshLevel + 1);
MeshLevelData& levelData = meshDistributor->getMeshLevelData(); MeshLevelData& levelData = meshDistributor->getMeshLevelData();
vector<const FiniteElemSpace*>& uniqueFe = meshDistributor->getFeSpaces();
if (subdomain == NULL) { if (subdomain == NULL) {
subdomain = new PetscSolverGlobalMatrix(); subdomain = new PetscSolverGlobalMatrix();
...@@ -278,42 +279,19 @@ namespace AMDiS { ...@@ -278,42 +279,19 @@ namespace AMDiS {
} }
} }
primalDofMap.init(levelData, primalDofMap.init(levelData, feSpaces, uniqueFe);
feSpaces, meshDistributor->getFeSpaces(), dualDofMap.init(levelData, feSpaces, uniqueFe, false);
true, true); localDofMap.init(levelData, feSpaces, uniqueFe, meshLevel != 0);
primalDofMap.setComputeMatIndex(true); lagrangeMap.init(levelData, feSpaces, uniqueFe);
dualDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
false, false);
dualDofMap.setComputeMatIndex(true);
if (meshLevel == 0) {
localDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
false, false);
localDofMap.setComputeMatIndex(true);
} else {
localDofMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
true, true);
localDofMap.setComputeMatIndex(true);
}
lagrangeMap.init(levelData,
feSpaces, meshDistributor->getFeSpaces(),
true, true);
lagrangeMap.setComputeMatIndex(true);
if (fullInterface != NULL)
interfaceDofMap.init(levelData, fullInterface);
if (fetiPreconditioner != FETI_NONE) { if (fetiPreconditioner != FETI_NONE) {
TEST_EXIT(meshLevel == 0) TEST_EXIT(meshLevel == 0)
("Dirichlet preconditioner not yet implemented for multilevel FETI-DP\n"); ("Dirichlet preconditioner not yet implemented for multilevel FETI-DP\n");
interiorDofMap.init(levelData, interiorDofMap.init(levelData, feSpaces, uniqueFe, false);
feSpaces, meshDistributor->getFeSpaces(),
false, false);
interiorDofMap.setComputeMatIndex(true);
} }
} }
...@@ -352,12 +330,20 @@ namespace AMDiS { ...@@ -352,12 +330,20 @@ namespace AMDiS {
else else
localDofMap.setDofComm(meshDistributor->getDofCommSd()); localDofMap.setDofComm(meshDistributor->getDofCommSd());
if (fullInterface != NULL) {
interfaceDofMap.clear();
interfaceDofMap.setDofComm(meshDistributor->getDofComm());
interfaceDofMap.setMpiComm(levelData.getMpiComm(0), 0);
}
for (unsigned int i = 0; i < meshDistributor->getFeSpaces().size(); i++) { for (unsigned int i = 0; i < meshDistributor->getFeSpaces().size(); i++) {
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(i); const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(i);
createPrimals(feSpace); createPrimals(feSpace);
createDuals(feSpace); createDuals(feSpace);
createInterfaceNodes(feSpace);
createIndexB(feSpace); createIndexB(feSpace);
} }
...@@ -368,6 +354,9 @@ namespace AMDiS { ...@@ -368,6 +354,9 @@ namespace AMDiS {
if (fetiPreconditioner != FETI_NONE) if (fetiPreconditioner != FETI_NONE)
interiorDofMap.update(); interiorDofMap.update();
if (fullInterface != NULL)
interfaceDofMap.update();
for (unsigned int i = 0; i < meshDistributor->getFeSpaces().size(); i++) { for (unsigned int i = 0; i < meshDistributor->getFeSpaces().size(); i++) {
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(i); const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(i);
createLagrange(feSpace); createLagrange(feSpace);
...@@ -435,6 +424,9 @@ namespace AMDiS { ...@@ -435,6 +424,9 @@ namespace AMDiS {
{ {
FUNCNAME("PetscSolverFeti::createPrimals()"); FUNCNAME("PetscSolverFeti::createPrimals()");
if (feSpace == fullInterface)
return;
// === Define all vertices on the interior boundaries of the macro mesh === // === Define all vertices on the interior boundaries of the macro mesh ===
// === to be primal variables. === // === to be primal variables. ===
...@@ -477,6 +469,9 @@ namespace AMDiS { ...@@ -477,6 +469,9 @@ namespace AMDiS {
{ {
FUNCNAME("PetscSolverFeti::createDuals()"); FUNCNAME("PetscSolverFeti::createDuals()");
if (feSpace == fullInterface)
return;
// === Create global index of the dual nodes on each rank. === // === Create global index of the dual nodes on each rank. ===
DofContainer allBoundaryDofs; DofContainer allBoundaryDofs;
...@@ -494,10 +489,32 @@ namespace AMDiS { ...@@ -494,10 +489,32 @@ namespace AMDiS {
} }
void PetscSolverFeti::createInterfaceNodes(const FiniteElemSpace *feSpace)
{
FUNCNAME("PetscSolverFeti::createInterfaceNodes()");
if (feSpace != fullInterface)
return;
DofContainer allBoundaryDofs;
meshDistributor->getAllBoundaryDofs(feSpace, meshLevel, allBoundaryDofs);
for (DofContainer::iterator it = allBoundaryDofs.begin();
it != allBoundaryDofs.end(); ++it)
if (meshDistributor->getDofMap()[feSpace].isRankDof(*it))
interfaceDofMap[feSpace].insertRankDof(*it);
else
interfaceDofMap[feSpace].insertNonRankDof(*it);
}
void PetscSolverFeti::createLagrange(const FiniteElemSpace *feSpace) void PetscSolverFeti::createLagrange(const FiniteElemSpace *feSpace)
{ {
FUNCNAME("PetscSolverFeti::createLagrange()"); FUNCNAME("PetscSolverFeti::createLagrange()");
if (feSpace == fullInterface)
return;
boundaryDofRanks[feSpace].clear(); boundaryDofRanks[feSpace].clear();
// Stores for all rank owned communication DOFs, if the counterpart is // Stores for all rank owned communication DOFs, if the counterpart is
...@@ -645,7 +662,8 @@ namespace AMDiS { ...@@ -645,7 +662,8 @@ namespace AMDiS {
for (int i = 0; i < admin->getUsedSize(); i++) { for (int i = 0; i < admin->getUsedSize(); i++) {
if (admin->isDofFree(i) == false && if (admin->isDofFree(i) == false &&
isPrimal(feSpace, i) == false && isPrimal(feSpace, i) == false &&
isDual(feSpace, i) == false) { isDual(feSpace, i) == false &&
isInterface(feSpace, i) == false) {
if (meshLevel == 0) { if (meshLevel == 0) {
localDofMap[feSpace].insertRankDof(i, nLocalInterior); localDofMap[feSpace].insertRankDof(i, nLocalInterior);
......
...@@ -108,9 +108,12 @@ namespace AMDiS { ...@@ -108,9 +108,12 @@ namespace AMDiS {
void createPrimals(const FiniteElemSpace *feSpace); void createPrimals(const FiniteElemSpace *feSpace);
/// Defines the set of dual variables and creates the global index of /// Defines the set of dual variables and creates the global index of
// dual variables. /// dual variables.
void createDuals(const FiniteElemSpace *feSpace); void createDuals(const FiniteElemSpace *feSpace);
///
void createInterfaceNodes(const FiniteElemSpace *feSpace);
/// Create Lagrange multiplier variables corresponding to the dual /// Create Lagrange multiplier variables corresponding to the dual
/// variables. /// variables.
void createLagrange(const FiniteElemSpace *feSpace); void createLagrange(const FiniteElemSpace *feSpace);
...@@ -192,6 +195,13 @@ namespace AMDiS { ...@@ -192,6 +195,13 @@ namespace AMDiS {
return dualDofMap[feSpace].isSet(dof); return dualDofMap[feSpace].isSet(dof);
} }
/// Checks whether a given DOF in a give FE space is an interface DOF.
inline bool isInterface(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof)
{
return interfaceDofMap[feSpace].isSet(dof);
}
protected: protected:
/// Mapping from primal DOF indices to a global index of primals. /// Mapping from primal DOF indices to a global index of primals.
ParallelDofMapping primalDofMap; ParallelDofMapping primalDofMap;
...@@ -199,6 +209,11 @@ namespace AMDiS { ...@@ -199,6 +209,11 @@ namespace AMDiS {
/// Mapping from dual DOF indices to a global index of duals. /// Mapping from dual DOF indices to a global index of duals.
ParallelDofMapping dualDofMap; ParallelDofMapping dualDofMap;
/// Mapping from interface DOF indices to a global index of interface
/// nodes. This is mainly used for Stokes-like solvers, where the pressure
/// interface nodes are neither primal nor dual.
ParallelDofMapping interfaceDofMap;
/// Index for each non primal DOF to the global index of B variables (thus, /// Index for each non primal DOF to the global index of B variables (thus,
/// all pure local variables). /// all pure local variables).
ParallelDofMapping localDofMap; ParallelDofMapping localDofMap;
...@@ -271,6 +286,8 @@ namespace AMDiS { ...@@ -271,6 +286,8 @@ namespace AMDiS {
int nGlobalOverallInterior; int nGlobalOverallInterior;
const FiniteElemSpace* fullInterface;
bool printTimings; bool printTimings;
}; };
......
...@@ -598,7 +598,7 @@ namespace AMDiS { ...@@ -598,7 +598,7 @@ namespace AMDiS {
if (!nnzInterior.dnnz || recvAllValues != 0 || alwaysCreateNnzStructure) { if (!nnzInterior.dnnz || recvAllValues != 0 || alwaysCreateNnzStructure) {
vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat); vector<const FiniteElemSpace*> feSpaces = getFeSpaces(mat);
interiorMap->setComputeMatIndex(true, !localMatrix); interiorMap->setComputeMatIndex(!localMatrix);
interiorMap->update(feSpaces); interiorMap->update(feSpaces);
nnzInterior.clear(); nnzInterior.clear();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment