Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konto der externen Nutzer:innen sind über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab. The administrators

Commit 3e2d84dc authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Setup for multilevel parallel dof mapping.

parent a6892347
......@@ -1200,7 +1200,7 @@ namespace AMDiS {
// === Run mesh partitioner to calculate a new mesh partitioning. ===
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap()));
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0)));
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART);
......@@ -1979,9 +1979,9 @@ namespace AMDiS {
MSG("------------- Debug information -------------\n");
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("FE space %d:\n", i);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs);
MSG(" nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[0]);
MSG(" nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[0]);
MSG(" rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[0]);
}
stringstream oss;
......@@ -2259,20 +2259,6 @@ namespace AMDiS {
}
DegreeOfFreedom MeshDistributor::mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof)
{
FUNCNAME("MeshDistributor::mapGlobalToLocal()");
for (DofMap::iterator it = dofMap[feSpace].getMap().begin();
it != dofMap[feSpace].getMap().end(); ++it)
if (it->second.global == dof)
return it->first;
return -1;
}
void MeshDistributor::serialize(ostream &out)
{
FUNCNAME("MeshDistributor::serialize()");
......
......@@ -157,13 +157,6 @@ namespace AMDiS {
return periodicMap;
}
/// Returns for a global index the DOF index in rank's subdomain. As there
/// is no direct data structure that stores this information, we have to
/// search for it in \ref dofMap. This is not very efficient and this
/// function should thus be used for debugging only.
DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof);
DofComm& getSendDofs()
{
return sendDofs;
......
......@@ -635,6 +635,11 @@ namespace AMDiS {
void ParallelDebug::printMapLocalGlobal(MeshDistributor &pdb, int rank)
{
FUNCNAME("ParallelDebug::printMapLocalGlobal()");
ERROR_EXIT("Rewrite this function!\n");
#if 0
if (rank == -1 || pdb.mpiRank == rank) {
const FiniteElemSpace *feSpace = pdb.feSpaces[0];
......@@ -662,6 +667,7 @@ namespace AMDiS {
cout << "------" << endl;
}
}
#endif
}
......
......@@ -20,47 +20,61 @@ namespace AMDiS {
void FeSpaceDofMap::clear()
{
dofMap.clear();
dofMap.resize(nLevel);
nonRankDofs.clear();
nRankDofs = 0;
nLocalDofs = 0;
nOverallDofs = 0;
rStartDofs = 0;
nonRankDofs.resize(nLevel);
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = 0;
nLocalDofs[i] = 0;
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
}
}
void FeSpaceDofMap::update()
{
FUNCNAME("FeSpaceDofMap::update()");
for (int i = 0; i < nLevel; i++) {
// === Compute local indices for all rank owned DOFs. ===
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs++;
for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it)
if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0)
it->second.local = nRankDofs[i]++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs = 0;
rStartDofs = 0;
mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
mpi::getDofNumbering(mpiComm, nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. ===
if (needGlobalMapping) {
computeGlobalMapping();
computeGlobalMapping(i);
if (hasNonLocalDofs)
computeNonLocalIndices();
computeNonLocalIndices(i);
}
}
}
void FeSpaceDofMap::computeGlobalMapping()
void FeSpaceDofMap::computeGlobalMapping(int level)
{
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
it->second.global = it->second.local + rStartDofs;
FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
for (DofMap::iterator it = dofMap[level].begin();
it != dofMap[level].end(); ++it)
it->second.global = it->second.local + rStartDofs[level];
}
void FeSpaceDofMap::computeNonLocalIndices()
void FeSpaceDofMap::computeNonLocalIndices(int level)
{
FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
......@@ -70,20 +84,22 @@ namespace AMDiS {
// === other ranks that also include this DOF. ===
StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
for (DofComm::Iterator it(*sendDofs, level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()].global);
if (dofMap[level].count(it.getDofIndex()) && !nonRankDofs[level].count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[level][it.getDofIndex()].global);
stdMpi.updateSendDataSize();
// === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs.count(it.getDofIndex())) {
if (nonRankDofs[level].count(it.getDofIndex())) {
recvFromRank = true;
break;
}
......@@ -101,12 +117,12 @@ namespace AMDiS {
// === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(*recvDofs, feSpace);
for (DofComm::Iterator it(*recvDofs, level, feSpace);
!it.end(); it.nextRank()) {
int i = 0;
for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs.count(it.getDofIndex()))
dofMap[it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
if (nonRankDofs[level].count(it.getDofIndex()))
dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(it.getRank())[i++];
}
}
......@@ -144,10 +160,12 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].clear();
nRankDofs = -1;
nLocalDofs = -1;
nOverallDofs = -1;
rStartDofs = -1;
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = -1;
nLocalDofs[i] = -1;
nOverallDofs[i] = -1;
rStartDofs[i] = -1;
}
dofToMatIndex.clear();
}
......@@ -180,56 +198,56 @@ namespace AMDiS {
}
int ParallelDofMapping::computeRankDofs()
int ParallelDofMapping::computeRankDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nRankDofs;
result += data[feSpaces[i]].nRankDofs[level];
}
return result;
}
int ParallelDofMapping::computeLocalDofs()
int ParallelDofMapping::computeLocalDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nLocalDofs;
result += data[feSpaces[i]].nLocalDofs[level];
}
return result;
}
int ParallelDofMapping::computeOverallDofs()
int ParallelDofMapping::computeOverallDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.nOverallDofs;
result += data.find(feSpaces[i])->second.nOverallDofs[level];
}
return result;
}
int ParallelDofMapping::computeStartDofs()
int ParallelDofMapping::computeStartDofs(int level)
{
FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.rStartDofs;
result += data.find(feSpaces[i])->second.rStartDofs[level];
}
return result;
......@@ -245,15 +263,18 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].update();
for (int i = 0; i < nLevel; i++) {
// Compute all numbers from this mappings.
nRankDofs = computeRankDofs();
nLocalDofs = computeLocalDofs();
nOverallDofs = computeOverallDofs();
rStartDofs = computeStartDofs();
nRankDofs[i] = computeRankDofs(i);
nLocalDofs[i] = computeLocalDofs(i);
nOverallDofs[i] = computeOverallDofs(i);
rStartDofs[i] = computeStartDofs(i);
// And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal);
computeMatIndex(needMatIndexFromGlobal, i);
}
}
......@@ -272,16 +293,18 @@ namespace AMDiS {
}
void ParallelDofMapping::computeMatIndex(bool globalIndex)
void ParallelDofMapping::computeMatIndex(bool globalIndex, int level)
{
FUNCNAME("ParallelDofMapping::computeMatIndex()");
TEST_EXIT(level == 0)("Not yet implemented for non-zero level!\n");
dofToMatIndex.clear();
// The offset is always added to the local matrix index. The offset for the
// DOFs in the first FE spaces is the smalled global index of a DOF that is
// owned by the rank.
int offset = rStartDofs;
int offset = rStartDofs[level];
// === Create the matrix indices for all component FE spaces. ===
......@@ -290,7 +313,7 @@ namespace AMDiS {
// Traverse all DOFs of the FE space and create for all rank owned DOFs
// a matrix index.
DofMap& dofMap = data[feSpaces[i]].getMap();
DofMap& dofMap = data[feSpaces[i]].getMap(level);
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
if (data[feSpaces[i]].isRankDof(it->first)) {
int globalMatIndex = it->second.local + offset;
......@@ -303,7 +326,7 @@ namespace AMDiS {
// Increase the offset for the next FE space by the number of DOFs owned
// by the rank in the current FE space.
offset += data[feSpaces[i]].nRankDofs;
offset += data[feSpaces[i]].nRankDofs[level];
// If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs)
......@@ -316,7 +339,7 @@ namespace AMDiS {
// === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpaces[i]);
for (DofComm::Iterator it(*sendDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs;
......@@ -330,14 +353,14 @@ namespace AMDiS {
stdMpi.send(it.getRank(), sendGlobalDofs);
}
for (DofComm::Iterator it(*recvDofs, feSpaces[i]);
for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
{
for (DofComm::Iterator it(*recvDofs, feSpaces[i]);
for (DofComm::Iterator it(*recvDofs, level, feSpaces[i]);
!it.end(); it.nextRank()) {
int counter = 0;
for (; !it.endDofIter(); it.nextDof()) {
......
......@@ -107,13 +107,17 @@ namespace AMDiS {
sendDofs(NULL),
recvDofs(NULL),
feSpace(NULL),
nLevel(1),
dofMap(1),
needGlobalMapping(false),
hasNonLocalDofs(false),
nRankDofs(0),
nLocalDofs(0),
nOverallDofs(0),
rStartDofs(0)
{}
nRankDofs(1),
nLocalDofs(1),
nOverallDofs(1),
rStartDofs(1)
{
clear();
}
/// Clears all data of the mapping.
void clear();
......@@ -122,9 +126,9 @@ namespace AMDiS {
/// global index must not be set.
MultiIndex& operator[](DegreeOfFreedom d)
{
TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");
TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
return dofMap[d];
return dofMap[0][d];
}
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
......@@ -133,12 +137,12 @@ namespace AMDiS {
{
FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1;
nLocalDofs++;
dofMap[0][dof0].local = dof1;
nLocalDofs[0]++;
if (dof1 != -1)
nRankDofs++;
nRankDofs[0]++;
}
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
......@@ -147,17 +151,17 @@ namespace AMDiS {
{
FUNCNAME("FeSpaceDofMap::insert()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap[0].count(dof0) == 0)("Should not happen!\n");
dofMap[dof0].local = dof1;
nLocalDofs++;
nonRankDofs.insert(dof0);
dofMap[0][dof0].local = dof1;
nLocalDofs[0]++;
nonRankDofs[0].insert(dof0);
}
/// Checks if a given DOF is in the DOF mapping.
bool isSet(DegreeOfFreedom dof)
bool isSet(DegreeOfFreedom dof, int level = 0)
{
return static_cast<bool>(dofMap.count(dof));
return static_cast<bool>(dofMap[level].count(dof));
}
/// Checks if a given DOF is a rank owned DOF of the DOF mapping. The DOF must
......@@ -165,19 +169,19 @@ namespace AMDiS {
/// meaningsless.
bool isRankDof(DegreeOfFreedom dof)
{
return !(static_cast<bool>(nonRankDofs.count(dof)));
return !(static_cast<bool>(nonRankDofs[0].count(dof)));
}
/// Returns number of DOFs in the mapping.
unsigned int size()
{
return dofMap.size();
return dofMap[0].size();
}
/// Returns the raw data of the mapping.
DofMap& getMap()
DofMap& getMap(int level)
{
return dofMap;
return dofMap[level];
}
/// Recomputes the mapping.
......@@ -211,11 +215,11 @@ namespace AMDiS {
private:
/// Computes a global mapping from the local one.
void computeGlobalMapping();
void computeGlobalMapping(int level);
/// Computes the global indices of all DOFs in the mapping that are not owned
/// by the rank.
void computeNonLocalIndices();
void computeNonLocalIndices(int level);
private:
/// MPI communicator object;
......@@ -228,11 +232,14 @@ namespace AMDiS {
/// correct DOF communicator in \ref sendDofs and \ref recvDofs.
const FiniteElemSpace *feSpace;
/// Number of mesh levels.
int nLevel;
/// Mapping data from DOF indices to local and global indices.
DofMap dofMap;
vector<DofMap> dofMap;
/// Set of all DOFs that are in mapping but are not owned by the rank.
std::set<DegreeOfFreedom> nonRankDofs;
vector<std::set<DegreeOfFreedom> > nonRankDofs;
/// If true, a global index mapping will be computed for all DOFs.
bool needGlobalMapping;
......@@ -241,9 +248,10 @@ namespace AMDiS {
/// by the rank. If the value is false, each rank contains only DOFs that
/// are also owned by this rank.
bool hasNonLocalDofs;
public:
///
int nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
vector<int> nRankDofs, nLocalDofs, nOverallDofs, rStartDofs;
};
......@@ -256,18 +264,23 @@ namespace AMDiS {
{
public:
ParallelDofMapping()
: mpiComm(NULL),
sendDofs(NULL),
: sendDofs(NULL),
recvDofs(NULL),
hasNonLocalDofs(false),
needMatIndex(false),
needMatIndexFromGlobal(false),
nLevel(1),
feSpaces(0),
nRankDofs(-1),
nLocalDofs(-1),
nOverallDofs(-1),
rStartDofs(-1)
{}
nRankDofs(1),
nLocalDofs(1),
nOverallDofs(1),
rStartDofs(1)
{
nRankDofs[0] = -1;
nLocalDofs[0] = -1;
nOverallDofs[0] = -1;
rStartDofs[0] = -1;
}
/** \brief Initialize the parallel DOF mapping.
*
......@@ -307,36 +320,36 @@ namespace AMDiS {
}
/// Returns \ref nRankDofs, thus the number of DOFs owned by the rank.
inline int getRankDofs()
inline int getRankDofs(int level)
{
TEST_EXIT_DBG(nRankDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(nRankDofs[level] >= 0)("Should not happen!\n");
return nRankDofs;
return nRankDofs[level];
}
/// Returns \ref nLocalDofs, thus the number of DOFs in ranks subdomain.
inline int getLocalDofs()
inline int getLocalDofs(int level)
{
TEST_EXIT_DBG(nLocalDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(nLocalDofs[level] >= 0)("Should not happen!\n");
return nLocalDofs;
return nLocalDofs[level];
}
/// Returns \ref nOverallDofs, thus the number of all DOFs in this mapping.
inline int getOverallDofs()
inline int getOverallDofs(int level)
{
TEST_EXIT_DBG(nOverallDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(nOverallDofs[level] >= 0)("Should not happen!\n");
return nOverallDofs;
return nOverallDofs[level];
}
/// Returns \ref rStartDofs, thus the smallest global index of a DOF that is
/// owned by the rank.
inline int getStartDofs()
inline int getStartDofs(int level)
{
TEST_EXIT_DBG(rStartDofs >= 0)("Should not happen!\n");
TEST_EXIT_DBG(rStartDofs[level] >= 0)("Should not happen!\n");
return rStartDofs;
return rStartDofs[level];
}
/// Update the mapping.
......@@ -361,7 +374,7 @@ namespace AMDiS {
TEST_EXIT_DBG(data[feSpaces[ithComponent]].isRankDof(d))
("Should not happen!\n");
return dofToMatIndex.get(ithComponent, d) - rStartDofs;
return dofToMatIndex.get(ithComponent, d) - rStartDofs[0];
}
// Writes all data of this object to an output stream.
......@@ -383,23 +396,23 @@ namespace AMDiS {
}
/// Compute local and global matrix indices.
void computeMatIndex(bool globalIndex);
void computeMatIndex(bool globalIndex, int level);
protected:
/// Insert a new FE space DOF mapping for a given FE space.
void addFeSpace(const FiniteElemSpace* feSpace);