Commit 3e2d84dc authored by Thomas Witkowski's avatar Thomas Witkowski

Setup for multilevel parallel dof mapping.

parent a6892347
......@@ -1200,7 +1200,7 @@ namespace AMDiS {
// === Run mesh partitioner to calculate a new mesh partitioning. ===
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap()));
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0)));
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART);
......@@ -1979,9 +1979,9 @@ namespace AMDiS {
MSG("------------- Debug information -------------\n");
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("FE space %d:\n", i);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[0]);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[0]);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[0]);
}
stringstream oss;
......@@ -2259,20 +2259,6 @@ namespace AMDiS {
}
DegreeOfFreedom MeshDistributor::mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof)
{
FUNCNAME("MeshDistributor::mapGlobalToLocal()");
for (DofMap::iterator it = dofMap[feSpace].getMap().begin();
it != dofMap[feSpace].getMap().end(); ++it)
if (it->second.global == dof)
return it->first;
return -1;
}
void MeshDistributor::serialize(ostream &out)
{
FUNCNAME("MeshDistributor::serialize()");
......
......@@ -157,13 +157,6 @@ namespace AMDiS {
return periodicMap;
}
/// Returns for a global index the DOF index in rank's subdomain. As there
/// is no direct data structure that stores this information, we have to
/// search for it in \ref dofMap. This is not very efficient and this
/// function should thus be used for debugging only.
DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof);
DofComm& getSendDofs()
{
return sendDofs;
......
......@@ -634,7 +634,12 @@ namespace AMDiS {
void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank)
{
{
FUNCNAME("ParallelDebug::printMapLocalGlobal()");
ERROR_EXIT("Rewrite this function!\n");
#if 0
if (rank == -1 || pdb.mpiRank == rank) {
const FiniteElemSpace *feSpace = pdb.feSpaces[0];
......@@ -662,6 +667,7 @@ namespace AMDiS {
cout << "------" << endl;
}
}
#endif
}
......
......@@ -20,47 +20,61 @@ namespace AMDiS {
void FeSpaceDofMap::clear()
{
dofMap.clear();
dofMap.resize(nLevel);
nonRankDofs.clear();
nRankDofs = 0;
nLocalDofs = 0;
nOverallDofs = 0;
rStartDofs = 0;
nonRankDofs.resize(nLevel);
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = 0;
nLocalDofs[i] = 0;
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
}
}
void FeSpaceDofMap::update()
{
// === Compute local indices for all rank owned DOFs. ===
FUNCNAME("FeSpaceDofMap::update()");
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs = 0;
rStartDofs = 0;
mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
// === If required, compute also the global indices. ===
for (int i = 0; i < nLevel; i++) {
if (needGlobalMapping) {
computeGlobalMapping();
// === Compute local indices for all rank owned DOFs. ===
if (hasNonLocalDofs)
computeNonLocalIndices();
for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it)
if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0)
it->second.local = nRankDofs[i]++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
mpi::getDofNumbering(mpiComm, nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. ===
if (needGlobalMapping) {
computeGlobalMapping(i);
if (hasNonLocalDofs)
computeNonLocalIndices(i);
}
}
}
void FeSpaceDofMap::computeGlobalMapping()
void FeSpaceDofMap::computeGlobalMapping(int level)
{
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
it->second.global = it->second.local + rStartDofs;
FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
for (DofMap::iterator it = dofMap[level].begin();
it != dofMap[level].end(); ++it)
it->second.global = it->second.local + rStartDofs[level];
}
void FeSpaceDofMap::computeNonLocalIndices()
void FeSpaceDofMap::computeNonLocalIndices(int level)
{
FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
......@@ -70,20 +84,22 @@ namespace AMDiS {
// === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
for (DofComm::Iterator it(*sendDofs, level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
if (dofMap[level].count(it.getDofIndex()) && !nonRankDofs[level].count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[level][it.getDofIndex()].global);
stdMpi.updateSendDataSize();
// === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs.count(it.getDofIndex())) {
if (nonRankDofs[level].count(it.getDofIndex())) {
recvFromRank = true;
break;
}
......@@ -101,12 +117,12 @@ namespace AMDiS {
// === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(*recvDofs, feSpace);
for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) {
int i = 0;
for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs.count(it.getDofIndex()))
dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
if (nonRankDofs[level].count(it.getDofIndex()))
dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
}
}
......@@ -144,10 +160,12 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].clear();
nRankDofs = -1;
nLocalDofs = -1;
nOverallDofs = -1;
rStartDofs = -1;
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = -1;
nLocalDofs[i] = -1;
nOverallDofs[i] = -1;
rStartDofs[i] = -1;
}
dofToMatIndex.clear();
}
......@@ -180,56 +198,56 @@ namespace AMDiS {
}
int ParallelDofMapping::computeRankDofs()
int ParallelDofMapping::computeRankDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nRankDofs;
result += data[feSpaces[i]].nRankDofs[level];
}
return result;
}
int ParallelDofMapping::computeLocalDofs()
int ParallelDofMapping::computeLocalDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nLocalDofs;
result += data[feSpaces[i]].nLocalDofs[level];
}
return result;
}
int ParallelDofMapping::computeOverallDofs()
int ParallelDofMapping::computeOverallDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.nOverallDofs;
result += data.find(feSpaces[i])->second.nOverallDofs[level];
}
return result;
}
int ParallelDofMapping::computeStartDofs()
int ParallelDofMapping::computeStartDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.rStartDofs;
result += data.find(feSpaces[i])->second.rStartDofs[level];
}
return result;
......@@ -245,15 +263,18 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].update();
// Compute all numbers from this mappings.
nRankDofs = computeRankDofs();
nLocalDofs = computeLocalDofs();
nOverallDofs = computeOverallDofs();
rStartDofs = computeStartDofs();
// And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal);
for (int i = 0; i < nLevel; i++) {
// Compute all numbers from this mappings.
nRankDofs[i] = computeRankDofs(i);
nLocalDofs[i] = computeLocalDofs(i);
nOverallDofs[i] = computeOverallDofs(i);
rStartDofs[i] = computeStartDofs(i);
// And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal, i);
}
}
......@@ -272,16 +293,18 @@ namespace AMDiS {
}
void ParallelDofMapping::computeMatIndex(bool globalIndex)
void ParallelDofMapping::computeMatIndex(bool globalIndex, int level)
{
FUNCNAME("ParallelDofMapping::computeMatIndex()");
TEST_EXIT(level == 0)("Not yet implemented for non-zero level!\n");
dofToMatIndex.clear();
// The offset is always added to the local matrix index. The offset for the
// DOFs in the first FE spaces is the smalled global index of a DOF that is
// owned by the rank.
int offset = rStartDofs;
int offset = rStartDofs[level];
// === Create the matrix indices for all component FE spaces. ===
......@@ -290,7 +313,7 @@ namespace AMDiS {
// Traverse all DOFs of the FE space and create for all rank owned DOFs
// a matrix index.
DofMap& dofMap = data[feSpaces[i]].getMap();
DofMap& dofMap = data[feSpaces[i]].getMap(level);
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
if (data[feSpaces[i]].isRankDof(it->first)) {
int globalMatIndex = it->second.local + offset;
......@@ -303,7 +326,7 @@ namespace AMDiS {
// Increase the offset for the next FE space by the number of DOFs owned
// by the rank in the current FE space.
offset += data[feSpaces[i]].nRankDofs;
offset += data[feSpaces[i]].nRankDofs[level];
// If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs)
......@@ -316,7 +339,7 @@ namespace AMDiS {
// === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpaces[i]);
for (DofComm::Iterator it(*sendDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs;
......@@ -330,14 +353,14 @@ namespace AMDiS {
stdMpi.send(it.getRank(), sendGlobalDofs);
}
for (DofComm::Iterator it(*recvDofs, feSpaces[i]);
for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
{
for (DofComm::Iterator it(*recvDofs, feSpaces[i]);
for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) {
int counter = 0;
for (; !it.endDofIter(); it.nextDof()) {
......
......@@ -107,13 +107,17 @@ namespace AMDiS {
sendDofs(NULL),
recvDofs(NULL),
feSpace(NULL),
nLevel(1),
dofMap(1),
needGlobalMapping(false),
hasNonLocalDofs(false),
nRankDofs(0),
nLocalDofs(0),
nOverallDofs(0),
rStartDofs(0)
{}
nRankDofs(1),
nLocalDofs(1),
nOverallDofs(1),
rStartDofs(1)
{
clear();
}
/// Clears all data of the mapping.
void clear();
......@@ -122,9 +126,9 @@ namespace AMDiS {
/// global index must not be set.
MultiIndex& operator[](DegreeOfFreedom d)
{
TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");
TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
return dofMap[d];
return dofMap[0][d];
}
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
......@@ -133,12 +137,12 @@ namespace AMDiS {
{
FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1;
nLocalDofs++;
dofMap[0][dof0].local = dof1;
nLocalDofs[0]++;
if (dof1 != -1)
nRankDofs++;
nRankDofs[0]++;
}
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
......@@ -147,17 +151,17 @@ namespace AMDiS {
{
FUNCNAME("FeSpaceDofMap::insert()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1;
nLocalDofs++;
nonRankDofs.insert(dof0);
dofMap[0][dof0].local = dof1;
nLocalDofs[0]++;
nonRankDofs[0].insert(dof0);
}
/// Checks if a given DOF is in the DOF mapping.
bool isSet(DegreeOfFreedom dof)
bool isSet(DegreeOfFreedom dof, int level = 0)
{
return static_cast<bool>(dofMap.count(dof));
return static_cast<bool>(dofMap[level].count(dof));
}
/// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
......@@ -165,19 +169,19 @@ namespace AMDiS {
/// meaningsless.
bool isRankDof(DegreeOfFreedom dof)
{
return !(static_cast<bool>(nonRankDofs.count(dof)));
return !(static_cast<bool>(nonRankDofs[0].count(dof)));
}
/// Returns number of DOFs in the mapping.
unsigned int size()
{
return dofMap.size();
return dofMap[0].size();
}
/// Returns the raw data of the mapping.
DofMap& getMap()
DofMap& getMap(int level)
{
return dofMap;
return dofMap[level];
}
/// Recomputes the mapping.
......@@ -211,11 +215,11 @@ namespace AMDiS {
private:
/// Computes a global mapping from the local one.
void computeGlobalMapping();
void computeGlobalMapping(int level);
/// Computes the global indices of all DOFs in the mapping that are not owned
/// by the rank.
void computeNonLocalIndices();
void computeNonLocalIndices(int level);
private:
/// MPI communicator object;
......@@ -228,11 +232,14 @@ namespace AMDiS {
/// correct DOF communicator in \ref sendDofs and \ref recvDofs.
const FiniteElemSpace *feSpace;
/// Number of mesh levels.
int nLevel;
/// Mapping data from DOF indices to local and global indices.
DofMap dofMap;
vector<DofMap> dofMap;
/// Set of all DOFs that are in mapping but are not owned by the rank.
std::set<DegreeOfFreedom> nonRankDofs;
vector<std::set<DegreeOfFreedom> > nonRankDofs;
/// If true, a global index mapping will be computed for all DOFs.
bool needGlobalMapping;
......@@ -241,9 +248,10 @@ namespace AMDiS {
/// by the rank. If the value is false, each rank contains only DOFs that
/// are also owned by this rank.
bool hasNonLocalDofs;
public:
///
int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
vector<int> nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
};
......@@ -256,18 +264,23 @@ namespace AMDiS {
{
public:
ParallelDofMapping()
: mpiComm(NULL),
sendDofs(NULL),
: sendDofs(NULL),
recvDofs(NULL),
hasNonLocalDofs(false),
needMatIndex(false),
needMatIndexFromGlobal(false),
nLevel(1),
feSpaces(0),
nRankDofs(-1),
nLocalDofs(-1),
nOverallDofs(-1),
rStartDofs(-1)
{}
nRankDofs(1),
nLocalDofs(1),
nOverallDofs(1),
rStartDofs(1)
{
nRankDofs[0] = -1;
nLocalDofs[0] = -1;
nOverallDofs[0] = -1;
rStartDofs[0] = -1;
}
/** \brief Initialize the parallel DOF mapping.
*
......@@ -307,36 +320,36 @@ namespace AMDiS {
}
/// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
inline int getRankDofs()
inline int getRankDofs(int level)
{
TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(nRankDofs[level] >= 0)("Should not happen!\n");
return nRankDofs;
return nRankDofs[level];
}
/// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
inline int getLocalDofs()
inline int getLocalDofs(int level)
{
TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(nLocalDofs[level] >= 0)("Should not happen!\n");
return nLocalDofs;
return nLocalDofs[level];
}
/// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
inline int getOverallDofs()
inline int getOverallDofs(int level)
{
TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(nOverallDofs[level] >= 0)("Should not happen!\n");
return nOverallDofs;
return nOverallDofs[level];
}
/// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
/// owned by the rank.
inline int getStartDofs()
inline int getStartDofs(int level)
{
TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(rStartDofs[level] >= 0)("Should not happen!\n");
return rStartDofs;
return rStartDofs[level];
}
/// Update the mapping.
......@@ -361,7 +374,7 @@ namespace AMDiS {
TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
("Should not happen!\n");
return dofToMatIndex.get(ithComponent, d) - rStartDofs;
return dofToMatIndex.get(ithComponent, d) - rStartDofs[0];
}
// Writes all data of this object to an output stream.
......@@ -383,23 +396,23 @@ namespace AMDiS {
}
/// Compute local and global matrix indices.
void computeMatIndex(bool globalIndex);
void computeMatIndex(bool globalIndex, int level);
protected:
/// Insert a new FE space DOF mapping for a given FE space.
void addFeSpace(const FiniteElemSpace* feSpace);
/// Compute \ref nRankDofs.
int computeRankDofs();
int computeRankDofs(int level);
/// Compute \ref nLocalDofs.
int computeLocalDofs();
int computeLocalDofs(int level);
/// Compute \ref nOverallDofs.
int computeOverallDofs();
int computeOverallDofs(int level);