Skip to content

GitLab

Commit 6e3ccc91 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

Backup my code before making big changes ....

parent e8831f17
Show whitespace changes
Inline Side-by-side
Showing with 200 additions and 158 deletions
AMDiS/src/parallel/FeSpaceMapping.cc
View file @ 6e3ccc91
......@@ -41,8 +41,6 @@ namespace AMDiS {
if (overlap)
computeNonLocalIndices();
computeMatIndex(0);
}
......@@ -91,52 +89,6 @@ namespace AMDiS {
}
void GlobalDofMap::computeMatIndex(int offset)
{
FUNCNAME("GlobalDofMap::computeMatIndex()");
map<DegreeOfFreedom, int> dofToMatIndex;
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
if (!nonRankDofs.count(it->first)) {
int globalMatIndex = it->second.local - rStartDofs + offset;
dofToMatIndex[it->first] = globalMatIndex;
}
}
if (!overlap)
return;
TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
("No communicator given!\n");
StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs;
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()))
sendGlobalDofs.push_back(dofToMatIndex[it.getDofIndex()]);
stdMpi.send(it.getRank(), sendGlobalDofs);
}
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
{
int i = 0;
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()))
dofToMatIndex[it.getDofIndex()] =
stdMpi.getRecvData(it.getRank())[i++];
}
}
void GlobalDofMap::print()
{
FUNCNAME("GlobalDofMap::print()");
......
AMDiS/src/parallel/FeSpaceMapping.h
View file @ 6e3ccc91
......@@ -26,6 +26,7 @@
#include "parallel/DofComm.h"
#include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h"
#include "parallel/StdMpi.h"
#ifndef AMDIS_FE_SPACE_MAPPING_H
#define AMDIS_FE_SPACE_MAPPING_H
......@@ -38,6 +39,53 @@ namespace AMDiS {
int local, global;
};
/** \brief
* Defines for each system component a mapping for sets of global DOF indices
* to global matrix indices. The mapping is defined for all DOFs in rank's
* subdomain. When periodic boundary conditions are used, then the mapping
* stores also information for the periodic associations of rank's DOF on
* periodic boundaries.
*/
class DofToMatIndex
{
public:
DofToMatIndex() {}
/// Reset the data structure.
inline void clear()
{
data.clear();
}
/// Add a new mapping.
inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
{
data[component][dof] = globalMatIndex;
}
/// Map a global DOF index to the global matrix index for a specific
/// system component number.
inline int get(int component, DegreeOfFreedom dof)
{
FUNCNAME("DofToMatIndex::get()");
TEST_EXIT_DBG(data.count(component))
("No mapping data for component %d available!\n", component);
TEST_EXIT_DBG(data[component].count(dof))
("Mapping for DOF %d in component %d does not exists!\n",
dof, component);
return data[component][dof];
}
private:
/// The mapping data. For each system component there is a specific map that
/// maps global DOF indices to global matrix indices.
map<int, map<DegreeOfFreedom, int> > data;
};
class GlobalDofMap
{
public:
......@@ -50,9 +98,9 @@ namespace AMDiS {
GlobalDofMap(MPI::Intracomm* m)
: mpiComm(m),
feSpace(NULL),
sendDofs(NULL),
recvDofs(NULL),
feSpace(NULL),
needGlobalMapping(false),
nRankDofs(0),
nLocalDofs(0),
......@@ -98,6 +146,11 @@ namespace AMDiS {
return static_cast<bool>(dofMap.count(dof));
}
bool isRankDof(DegreeOfFreedom dof)
{
return !(static_cast<bool>(nonRankDofs.count(dof)));
}
unsigned int size()
{
return dofMap.size();
......@@ -114,8 +167,6 @@ namespace AMDiS {
void computeNonLocalIndices();
void computeMatIndex(int offset);
void print();
void setFeSpace(const FiniteElemSpace *fe)
......@@ -128,20 +179,24 @@ namespace AMDiS {
overlap = b;
}
void setDofComm(DofComm &pSend, DofComm &pRecv)
void setNeedGlobalMapping(bool b)
{
sendDofs = &pSend;
recvDofs = &pRecv;
needGlobalMapping = b;
}
void setNeedGlobalMapping(bool b)
void setDofComm(DofComm &pSend, DofComm &pRecv)
{
needGlobalMapping = b;
sendDofs = &pSend;
recvDofs = &pRecv;
}
private:
MPI::Intracomm* mpiComm;
DofComm *sendDofs;
DofComm *recvDofs;
const FiniteElemSpace *feSpace;
///
......@@ -149,10 +204,6 @@ namespace AMDiS {
std::set<DegreeOfFreedom> nonRankDofs;
DofComm *sendDofs;
DofComm *recvDofs;
bool needGlobalMapping;
public:
///
......@@ -168,6 +219,9 @@ namespace AMDiS {
public:
FeSpaceData()
: mpiComm(NULL),
sendDofs(NULL),
recvDofs(NULL),
overlap(false),
feSpaces(0),
nRankDofs(-1),
nOverallDofs(-1),
......@@ -273,15 +327,18 @@ namespace AMDiS {
void init(MPI::Intracomm *m,
vector<const FiniteElemSpace*> &fe,
bool needGlobalMapping)
bool needGlobalMapping,
bool ol)
{
mpiComm = m;
feSpaces = fe;
overlap = ol;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
feSpacesUnique.insert(feSpaces[i]);
addFeSpace(feSpaces[i]);
data[feSpaces[i]].setNeedGlobalMapping(needGlobalMapping);
data[feSpaces[i]].setOverlap(overlap);
}
}
......@@ -295,7 +352,70 @@ namespace AMDiS {
nLocalDofs = getLocalDofs(feSpaces);
nOverallDofs = getOverallDofs(feSpaces);
rStartDofs = getStartDofs(feSpaces);
computeMatIndex();
}
void computeMatIndex()
{
dofToMatIndex.clear();
int offset = rStartDofs;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
map<DegreeOfFreedom, MultiIndex>& dofMap = data[feSpaces[i]].getMap();
typedef map<DegreeOfFreedom, MultiIndex>::iterator ItType;
for (ItType it = dofMap.begin(); it != dofMap.end(); ++it) {
if (data[feSpaces[i]].isRankDof(it->first)) {
int globalMatIndex =
it->second.local - data[feSpaces[i]].rStartDofs + offset;
dofToMatIndex.add(i, it->first, globalMatIndex);
}
}
if (!overlap)
continue;
TEST_EXIT_DBG(sendDofs != NULL && recvDofs != NULL)
("No communicator given!\n");
StdMpi<vector<DegreeOfFreedom> > stdMpi(*mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpaces[i]);
!it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs;
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()))
sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
stdMpi.send(it.getRank(), sendGlobalDofs);
}
for (DofComm::Iterator it(*recvDofs, feSpaces[i]);
!it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
{
int counter = 0;
for (DofComm::Iterator it(*recvDofs, feSpaces[i]);
!it.end(); it.nextRank()) {
for (; !it.endDofIter(); it.nextDof()) {
if (dofMap.count(it.getDofIndex())) {
DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[counter++];
dofToMatIndex.add(i, it.getDofIndex(), d);
}
}
}
}
offset += data[feSpaces[i]].nRankDofs;
}
}
inline int mapLocal(int index, int ithFeSpace)
{
......@@ -335,9 +455,25 @@ namespace AMDiS {
return -1;
}
void setDofComm(DofComm &pSend, DofComm &pRecv)
{
sendDofs = &pSend;
recvDofs = &pRecv;
for (std::set<const FiniteElemSpace*>::iterator it = feSpacesUnique.begin();
it != feSpacesUnique.end(); ++it)
data[*it].setDofComm(pSend, pRecv);
}
private:
MPI::Intracomm* mpiComm;
DofComm *sendDofs;
DofComm *recvDofs;
bool overlap;
map<const FiniteElemSpace*, T> data;
vector<const FiniteElemSpace*> feSpaces;
......@@ -345,6 +481,10 @@ namespace AMDiS {
std::set<const FiniteElemSpace*> feSpacesUnique;
int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
/// Mapping from global DOF indices to global matrix indices under
/// consideration of possibly multiple components.
DofToMatIndex dofToMatIndex;
};
}
......
AMDiS/src/parallel/PetscSolverFeti.cc
View file @ 6e3ccc91
......@@ -224,6 +224,11 @@ namespace AMDiS {
createIndexB(feSpace);
}
primalDofMap.setDofComm(meshDistributor->getSendDofs(),
meshDistributor->getRecvDofs());
lagrangeMap.setDofComm(meshDistributor->getSendDofs(),
meshDistributor->getRecvDofs());
primalDofMap.update();
dualDofMap.update();
lagrangeMap.update();
......@@ -242,6 +247,10 @@ namespace AMDiS {
MSG("nRankLagrange = %d nOverallLagrange = %d\n",
lagrangeMap[feSpace].nRankDofs,
lagrangeMap[feSpace].nOverallDofs);
TEST_EXIT_DBG(localDofMap[feSpace].size() + primalDofMap[feSpace].size() ==
static_cast<unsigned int>(feSpace->getAdmin()->getUsedDofs()))
("Should not happen!\n");
}
}
......@@ -272,10 +281,6 @@ namespace AMDiS {
primalDofMap[feSpace].insertRankDof(*it);
else
primalDofMap[feSpace].insert(*it);
primalDofMap[feSpace].setOverlap(true);
primalDofMap[feSpace].setDofComm(meshDistributor->getSendDofs(),
meshDistributor->getRecvDofs());
}
......@@ -283,6 +288,22 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverFeti::createDuals()");
// === Create global index of the dual nodes on each rank. ===
DofContainer allBoundaryDofs;
meshDistributor->getAllBoundaryDofs(feSpace, allBoundaryDofs);
for (DofContainer::iterator it = allBoundaryDofs.begin();
it != allBoundaryDofs.end(); ++it)
if (!isPrimal(feSpace, **it))
dualDofMap[feSpace].insertRankDof(**it);
}
void PetscSolverFeti::createLagrange(const FiniteElemSpace *feSpace)
{
FUNCNAME("PetscSolverFeti::createLagrange()");
// === Create for each dual node that is owned by the rank, the set ===
// === of ranks that contain this node (denoted by W(x_j)). ===
......@@ -336,21 +357,6 @@ namespace AMDiS {
stdMpi.getRecvData(it.getRank())[i++];
}
// === Create global index of the dual nodes on each rank. ===
DofContainer allBoundaryDofs;
meshDistributor->getAllBoundaryDofs(feSpace, allBoundaryDofs);
for (DofContainer::iterator it = allBoundaryDofs.begin();
it != allBoundaryDofs.end(); ++it)
if (!isPrimal(feSpace, **it))
dualDofMap[feSpace].insertRankDof(**it);
}
void PetscSolverFeti::createLagrange(const FiniteElemSpace *feSpace)
{
FUNCNAME("PetscSolverFeti::createLagrange()");
// === Reserve for each dual node, on the rank that owns this node, the ===
// === appropriate number of Lagrange constraints. ===
......@@ -367,9 +373,6 @@ namespace AMDiS {
}
}
lagrangeMap[feSpace].nRankDofs = nRankLagrange;
lagrangeMap[feSpace].setOverlap(true);
lagrangeMap[feSpace].setDofComm(meshDistributor->getSendDofs(),
meshDistributor->getRecvDofs());
}
......@@ -383,21 +386,16 @@ namespace AMDiS {
// === the global index of all B nodes, insert all interior nodes first, ===
// === without defining a correct index. ===
nLocalInterior = 0;
nLocalInterior[feSpace] = 0;
for (int i = 0; i < admin->getUsedSize(); i++) {
if (admin->isDofFree(i) == false &&
isPrimal(feSpace, i) == false &&
dualDofMap[feSpace].isSet(i) == false) {
localDofMap[feSpace].insertRankDof(i, nLocalInterior);
nLocalInterior++;
localDofMap[feSpace].insertRankDof(i, nLocalInterior[feSpace]);
nLocalInterior[feSpace]++;
}
}
TEST_EXIT_DBG(nLocalInterior + primalDofMap[feSpace].size() +
dualDofMap[feSpace].size() ==
static_cast<unsigned int>(admin->getUsedDofs()))
("Should not happen!\n");
// === And finally, add the global indicies of all dual nodes. ===
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dualDofMap[feSpace].getMap().begin();
......@@ -446,7 +444,6 @@ namespace AMDiS {
// Set the constraint for all components of the system.
for (int k = 0; k < nComponents; k++) {
int rowIndex = index * nComponents + k;
// int colIndex = it->second * nComponents + k;
int colIndex =
(localDofMap[feSpace].rStartDofs +
localDofMap[feSpace][it->first].local) * nComponents + k;
......@@ -470,8 +467,6 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverFeti::createSchurPrimal()");
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(0);
if (schurPrimalSolver == 0) {
MSG("Create iterative schur primal solver!\n");
......@@ -538,8 +533,10 @@ namespace AMDiS {
int maxLocalPrimal = mat_b_primal_cols.size();
mpi::globalMax(maxLocalPrimal);
TEST_EXIT(mat_b_primal_cols.size() == primalDofMap.getLocalDofs())
TEST_EXIT(static_cast<int>(mat_b_primal_cols.size()) ==
primalDofMap.getLocalDofs())
("Should not happen!\n");
for (map<int, vector<pair<int, double> > >::iterator it = mat_b_primal_cols.begin();
it != mat_b_primal_cols.end(); ++it) {
Vec tmpVec;
......@@ -862,10 +859,10 @@ namespace AMDiS {
// === Create all sets and indices. ===
primalDofMap.init(mpiComm, feSpaces, true);
dualDofMap.init(mpiComm, feSpaces, false);
lagrangeMap.init(mpiComm, feSpaces, true);
localDofMap.init(mpiComm, feSpaces, false);
primalDofMap.init(mpiComm, feSpaces, true, true);
dualDofMap.init(mpiComm, feSpaces, false, false);
lagrangeMap.init(mpiComm, feSpaces, true, true);
localDofMap.init(mpiComm, feSpaces, false, false);
updateDofData();
......@@ -958,7 +955,7 @@ namespace AMDiS {
for (cursor_type cursor = begin<row>((*mat)[i][j]->getBaseMatrix()),
cend = end<row>((*mat)[i][j]->getBaseMatrix()); cursor != cend; ++cursor) {
bool rowPrimal = primalDofMap[feSpace].isSet(*cursor);
bool rowPrimal = isPrimal(feSpace, *cursor);
cols.clear();
colsOther.clear();
......@@ -1002,26 +999,26 @@ namespace AMDiS {
int rowIndex = localDofMap[feSpace][*cursor].local;
int colIndex = localDofMap[feSpace][col(*icursor)].local;
if (rowIndex < nLocalInterior) {
if (colIndex < nLocalInterior) {
if (rowIndex < nLocalInterior[feSpace]) {
if (colIndex < nLocalInterior[feSpace]) {
int colIndex2 = localDofMap.mapLocal(col(*icursor), j);
colsLocal.push_back(colIndex2);
valuesLocal.push_back(value(*icursor));
} else {
int colIndex2 =
(localDofMap[feSpace][col(*icursor)].local - nLocalInterior) *
(localDofMap[feSpace][col(*icursor)].local - nLocalInterior[feSpace]) *
nComponents + j;
colsLocalOther.push_back(colIndex2);
valuesLocalOther.push_back(value(*icursor));
}
} else {
if (colIndex < nLocalInterior) {
if (colIndex < nLocalInterior[feSpace]) {
int colIndex2 = localDofMap.mapLocal(col(*icursor), j);
colsLocalOther.push_back(colIndex2);
valuesLocalOther.push_back(value(*icursor));
} else {
int colIndex2 = (localDofMap[feSpace][col(*icursor)].local - nLocalInterior) *
int colIndex2 = (localDofMap[feSpace][col(*icursor)].local - nLocalInterior[feSpace]) *
nComponents + j;
colsLocal.push_back(colIndex2);
......@@ -1082,7 +1079,7 @@ namespace AMDiS {
{
int rowIndex = localDofMap[feSpace][*cursor].local;
if (rowIndex < nLocalInterior) {
if (rowIndex < nLocalInterior[feSpace]) {
int rowIndex2 = localDofMap[feSpace][*cursor].local * nComponents + i;
MatSetValues(mat_interior_interior, 1, &rowIndex2, colsLocal.size(),
......@@ -1093,7 +1090,7 @@ namespace AMDiS {
&(colsLocalOther[0]), &(valuesLocalOther[0]), INSERT_VALUES);
} else {
int rowIndex2 =
(localDofMap[feSpace][*cursor].local - nLocalInterior) * nComponents + i;
(localDofMap[feSpace][*cursor].local - nLocalInterior[feSpace]) * nComponents + i;
MatSetValues(mat_duals_duals, 1, &rowIndex2, colsLocal.size(),
&(colsLocal[0]), &(valuesLocal[0]), INSERT_VALUES);
......@@ -1110,9 +1107,9 @@ namespace AMDiS {
{
int rowIndex = localDofMap[feSpace][*cursor].local;
if (rowIndex >= nLocalInterior) {
if (rowIndex >= nLocalInterior[feSpace]) {
int rowIndex2 =
(localDofMap[feSpace][*cursor].local - nLocalInterior) * nComponents + i;
(localDofMap[feSpace][*cursor].local - nLocalInterior[feSpace]) * nComponents + i;
MatSetValues(mat_duals_duals, 1, &rowIndex2, colsLocal.size(),
&(colsLocal[0]), &(valuesLocal[0]), INSERT_VALUES);
......
AMDiS/src/parallel/PetscSolverFeti.h
View file @ 6e3ccc91
......@@ -235,7 +235,7 @@ namespace AMDiS {
KSP ksp_interior;
int nLocalInterior;
map<const FiniteElemSpace*, int> nLocalInterior;
};
}
......
AMDiS/src/parallel/PetscSolverGlobalMatrix.h
View file @ 6e3ccc91
......@@ -26,59 +26,12 @@
#include <boost/tuple/tuple.hpp>
#include "AMDiS_fwd.h"
#include "parallel/PetscSolver.h"
#include "parallel/FeSpaceMapping.h"
namespace AMDiS {
using namespace std;
/** \brief
* Defines for each system component a mapping for sets of global DOF indices
* to global matrix indices. The mapping is defined for all DOFs in rank's
* subdomain. When periodic boundary conditions are used, then the mapping
* stores also information for the periodic associations of rank's DOF on
* periodic boundaries.
*/
class DofToMatIndex
{
public:
DofToMatIndex() {}
/// Reset the data structure.
inline void clear()
{
data.clear();
}
/// Add a new mapping.
inline void add(int component, DegreeOfFreedom dof, int globalMatIndex)
{
data[component][dof] = globalMatIndex;
}
/// Map a global DOF index to the global matrix index for a specific
/// system component number.
inline int get(int component, DegreeOfFreedom dof)
{
FUNCNAME("DofToMatIndex::get()");
TEST_EXIT_DBG(data.count(component))
("No mapping data for component %d available!\n", component);
TEST_EXIT_DBG(data[component].count(dof))
("Mapping for DOF %d in component %d does not exists!\n",
dof, component);
return data[component][dof];
}
private:
/// The mapping data. For each system component there is a specific map that