Commit 235f1f39 authored by Thomas Witkowski's avatar Thomas Witkowski

Removed level hierarchie in parallel dof mapping, is not necessary.

parent 346f9163
......@@ -238,7 +238,7 @@ namespace AMDiS {
if (condition->applyBoundaryCondition()) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if (dofMap->isRankDof(rowIndices[i], 0))
if (dofMap->isRankDof(rowIndices[i]))
applyDBCs.insert(static_cast<int>(row));
#else
applyDBCs.insert(static_cast<int>(row));
......
......@@ -248,7 +248,7 @@ namespace AMDiS {
{
TEST_EXIT_DBG(dofMap)("No rank dofs set!\n");
return dofMap->isRankDof(dof, 0);
return dofMap->isRankDof(dof);
}
#endif
......
......@@ -1246,7 +1246,7 @@ namespace AMDiS {
// === Run mesh partitioner to calculate a new mesh partitioning. ===
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap(0)));
partitioner->setLocalGlobalDofMap(&(dofMap[feSpaces[0]].getMap()));
bool partitioningSucceed =
partitioner->partition(elemWeights, ADAPTIVE_REPART);
......@@ -1654,23 +1654,19 @@ namespace AMDiS {
MSG("| number of levels: %d\n", nLevels);
MSG("| number of FE spaces: %d\n", feSpaces.size());
for (int level = 0; level < nLevels; level++) {
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| level = %d FE space = %d:\n", level, i);
MSG("| nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs[level]);
MSG("| nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs[level]);
MSG("| rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs[level]);
}
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| FE space = %d:\n", i);
MSG("| nRankDofs = %d\n", dofMap[feSpaces[i]].nRankDofs);
MSG("| nOverallDofs = %d\n", dofMap[feSpaces[i]].nOverallDofs);
MSG("| rStartDofs = %d\n", dofMap[feSpaces[i]].rStartDofs);
}
if (nLevels > 1) {
for (int level = 0; level < nLevels; level++) {
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| level = %d FE space = %d:\n", level, i);
MSG("| nRankDofs = %d\n", dofMapSd[feSpaces[i]].nRankDofs[level]);
MSG("| nOverallDofs = %d\n", dofMapSd[feSpaces[i]].nOverallDofs[level]);
MSG("| rStartDofs = %d\n", dofMapSd[feSpaces[i]].rStartDofs[level]);
}
for (unsigned int i = 0; i < feSpaces.size(); i++) {
MSG("| FE space = %d:\n", i);
MSG("| nRankDofs = %d\n", dofMapSd[feSpaces[i]].nRankDofs);
MSG("| nOverallDofs = %d\n", dofMapSd[feSpaces[i]].nOverallDofs);
MSG("| rStartDofs = %d\n", dofMapSd[feSpaces[i]].rStartDofs);
}
}
......@@ -1704,23 +1700,19 @@ namespace AMDiS {
sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
// === Traverse interior boundaries and get all DOFs on them. ===
int nLevels = levelData.getLevelNumber();
for (int level = 0; level < nLevels; level++) {
DofContainerSet nonRankDofs;
for (DofComm::Iterator it(dcom.getRecvDofs(), level, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof());
for (unsigned int i = 0; i < rankDofs.size(); i++)
if (nonRankDofs.count(rankDofs[i]) == 0)
dmap[feSpace].insertRankDof(level, *(rankDofs[i]));
for (DofContainerSet::iterator it = nonRankDofs.begin();
it != nonRankDofs.end(); ++it)
dmap[feSpace].insertNonRankDof(level, **it);
}
DofContainerSet nonRankDofs;
for (DofComm::Iterator it(dcom.getRecvDofs(), 0, feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
nonRankDofs.insert(it.getDof());
for (unsigned int i = 0; i < rankDofs.size(); i++)
if (nonRankDofs.count(rankDofs[i]) == 0)
dmap[feSpace].insertRankDof(*(rankDofs[i]));
for (DofContainerSet::iterator it = nonRankDofs.begin();
it != nonRankDofs.end(); ++it)
dmap[feSpace].insertNonRankDof(**it);
}
......@@ -1782,8 +1774,8 @@ namespace AMDiS {
BoundaryType type = bound.type;
for (unsigned int j = 0; j < dofs0.size(); j++) {
DegreeOfFreedom globalDof0 = dofMap[feSpace][0][*(dofs0[j])].global;
DegreeOfFreedom globalDof1 = dofMap[feSpace][0][*(dofs1[j])].global;
DegreeOfFreedom globalDof0 = dofMap[feSpace][*(dofs0[j])].global;
DegreeOfFreedom globalDof1 = dofMap[feSpace][*(dofs1[j])].global;
if (!periodicMap.isPeriodicOnBound(feSpace, type, globalDof0))
periodicMap.add(feSpace, type, globalDof0, globalDof1);
......@@ -1808,7 +1800,7 @@ namespace AMDiS {
// Send the global indices to the rank on the other side.
stdMpi.getSendData(it->first).reserve(dofs.size());
for (unsigned int i = 0; i < dofs.size(); i++)
stdMpi.getSendData(it->first).push_back(dofMap[feSpace][0][*(dofs[i])].global);
stdMpi.getSendData(it->first).push_back(dofMap[feSpace][*(dofs[i])].global);
// Receive from this rank the same number of dofs.
stdMpi.recv(it->first, dofs.size());
......@@ -1834,7 +1826,7 @@ namespace AMDiS {
// Added the received DOFs to the mapping.
for (unsigned int i = 0; i < dofs.size(); i++) {
int globalDofIndex = dofMap[feSpace][0][*(dofs[i])].global;
int globalDofIndex = dofMap[feSpace][*(dofs[i])].global;
int mapGlobalDofIndex = stdMpi.getRecvData(it->first)[i];
BoundaryType type = types[i];
......@@ -1867,7 +1859,7 @@ namespace AMDiS {
boundIt->rankObj.el->getAllDofs(feSpace, boundIt->rankObj, dofs);
for (unsigned int i = 0; i < dofs.size(); i++) {
DegreeOfFreedom globalDof = dofMap[feSpace][0][*dofs[i]].global;
DegreeOfFreedom globalDof = dofMap[feSpace][*dofs[i]].global;
std::set<BoundaryType>& assoc =
periodicMap.getAssociations(feSpace, globalDof);
......
......@@ -151,6 +151,11 @@ namespace AMDiS {
return dofMap;
}
inline ParallelDofMapping& getDofMapSd()
{
return dofMapSd;
}
/// Returns the periodic mapping handler, \ref periodicMap.
inline PeriodicMap& getPeriodicMap()
{
......
......@@ -480,7 +480,7 @@ namespace AMDiS {
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) {
coordsToIndex[(*it)] = pdb.dofMap[feSpace][0][it.getDOFIndex()].global;
coordsToIndex[(*it)] = pdb.dofMap[feSpace][it.getDOFIndex()].global;
// MSG(" CHECK FOR DOF %d AT COORDS %f %f %f\n",
// coordsToIndex[(*it)], (*it)[0], (*it)[1], (*it)[2]);
}
......@@ -794,8 +794,8 @@ namespace AMDiS {
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) {
file << it.getDOFIndex() << " "
<< pdb.dofMap[feSpace][0][it.getDOFIndex()].global << " "
<< pdb.dofMap[feSpace].isRankDof(it.getDOFIndex(), 0);
<< pdb.dofMap[feSpace][it.getDOFIndex()].global << " "
<< pdb.dofMap[feSpace].isRankDof(it.getDOFIndex());
for (int i = 0; i < pdb.mesh->getDim(); i++)
file << " " << (*it)[i];
file << "\n";
......
......@@ -19,22 +19,14 @@ namespace AMDiS {
void FeSpaceDofMap::clear()
{
int nLevel = levelData->getLevelNumber();
dofMap.clear();
dofMap.resize(nLevel);
nonRankDofs.clear();
nonRankDofs.resize(nLevel);
nRankDofs.clear();
nLocalDofs.clear();
nOverallDofs.clear();
rStartDofs.clear();
nRankDofs.resize(nLevel, 0);
nLocalDofs.resize(nLevel, 0);
nOverallDofs.resize(nLevel, 0);
rStartDofs.resize(nLevel, 0);
nRankDofs = 0;
nLocalDofs = 0;
nOverallDofs = 0;
rStartDofs = 0;
}
......@@ -42,48 +34,44 @@ namespace AMDiS {
{
FUNCNAME("FeSpaceDofMap::update()");
int nLevel = levelData->getLevelNumber();
for (int i = 0; i < nLevel; i++) {
// === Compute local indices for all rank owned DOFs. ===
for (DofMap::iterator it = dofMap[i].begin(); it != dofMap[i].end(); ++it)
if (it->second.local == -1 && nonRankDofs[i].count(it->first) == 0)
it->second.local = nRankDofs[i]++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs[i] = 0;
rStartDofs[i] = 0;
mpi::getDofNumbering(mpiComm,
nRankDofs[i], rStartDofs[i], nOverallDofs[i]);
// === If required, compute also the global indices. ===
// === Compute local indices for all rank owned DOFs. ===
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (it->second.local == -1 && nonRankDofs.count(it->first) == 0)
it->second.local = nRankDofs++;
// === Compute number of local and global DOFs in the mapping. ===
nOverallDofs = 0;
rStartDofs = 0;
mpi::getDofNumbering(mpiComm, nRankDofs, rStartDofs, nOverallDofs);
// === If required, compute also the global indices. ===
if (needGlobalMapping) {
computeGlobalMapping();
if (needGlobalMapping) {
computeGlobalMapping(i);
if (hasNonLocalDofs)
computeNonLocalIndices(i);
}
if (hasNonLocalDofs)
computeNonLocalIndices();
}
}
void FeSpaceDofMap::computeGlobalMapping(int level)
void FeSpaceDofMap::computeGlobalMapping()
{
FUNCNAME("FeSpaceDofMap::computeGlobalMapping()");
for (DofMap::iterator it = dofMap[level].begin();
it != dofMap[level].end(); ++it)
it->second.global = it->second.local + rStartDofs[level];
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
it->second.global = it->second.local + rStartDofs;
}
void FeSpaceDofMap::computeNonLocalIndices(int level)
void FeSpaceDofMap::computeNonLocalIndices()
{
FUNCNAME("FeSpaceDofMap::computeNonLocalIndices()");
TEST_EXIT_DBG(dofComm)("No DOF communicator defined!\n");
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;
// === Send all global indices of DOFs that are owned by the rank to all ===
......@@ -91,17 +79,17 @@ namespace AMDiS {
StdMpi<vector<int> > stdMpi(mpiComm);
for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpace);
for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpace);
!it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
rank = levelData->mapRank(rank, 0, meshLevel);
for (; !it.endDofIter(); it.nextDof())
if (dofMap[level].count(it.getDofIndex()) &&
!nonRankDofs[level].count(it.getDofIndex()))
if (dofMap.count(it.getDofIndex()) &&
!nonRankDofs.count(it.getDofIndex()))
stdMpi.getSendData(rank).
push_back(dofMap[level][it.getDofIndex()].global);
push_back(dofMap[it.getDofIndex()].global);
}
stdMpi.updateSendDataSize();
......@@ -109,11 +97,11 @@ namespace AMDiS {
// === Check from which ranks this rank must receive some data. ===
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpace);
for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs[level].count(it.getDofIndex())) {
if (nonRankDofs.count(it.getDofIndex())) {
recvFromRank = true;
break;
}
......@@ -136,7 +124,7 @@ namespace AMDiS {
// === And set the global indices for all DOFs that are not owned by rank. ===
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpace);
for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpace);
!it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
......@@ -144,8 +132,8 @@ namespace AMDiS {
int i = 0;
for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs[level].count(it.getDofIndex()))
dofMap[level][it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
if (nonRankDofs.count(it.getDofIndex()))
dofMap[it.getDofIndex()].global = stdMpi.getRecvData(rank)[i++];
}
}
......@@ -185,20 +173,11 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].clear();
int nLevel = levelData->getLevelNumber();
nRankDofs.resize(nLevel);
nLocalDofs.resize(nLevel);
nOverallDofs.resize(nLevel);
rStartDofs.resize(nLevel);
dofToMatIndex.resize(nLevel);
for (int i = 0; i < nLevel; i++) {
nRankDofs[i] = -1;
nLocalDofs[i] = -1;
nOverallDofs[i] = -1;
rStartDofs[i] = -1;
dofToMatIndex[i].clear();
}
nRankDofs = -1;
nLocalDofs = -1;
nOverallDofs = -1;
rStartDofs = -1;
dofToMatIndex.clear();
}
......@@ -239,56 +218,56 @@ namespace AMDiS {
}
int ParallelDofMapping::computeRankDofs(int level)
int ParallelDofMapping::computeRankDofs()
{
FUNCNAME("ParallelDofMapping::computeRankDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nRankDofs[level];
result += data[feSpaces[i]].nRankDofs;
}
return result;
}
int ParallelDofMapping::computeLocalDofs(int level)
int ParallelDofMapping::computeLocalDofs()
{
FUNCNAME("ParallelDofMapping::computeLocalDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data[feSpaces[i]].nLocalDofs[level];
result += data[feSpaces[i]].nLocalDofs;
}
return result;
}
int ParallelDofMapping::computeOverallDofs(int level)
int ParallelDofMapping::computeOverallDofs()
{
FUNCNAME("ParallelDofMapping::computeOverallDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.nOverallDofs[level];
result += data.find(feSpaces[i])->second.nOverallDofs;
}
return result;
}
int ParallelDofMapping::computeStartDofs(int level)
int ParallelDofMapping::computeStartDofs()
{
FUNCNAME("ParallelDofMapping::computeStartDofs()");
int result = 0;
for (unsigned int i = 0; i < feSpaces.size(); i++) {
TEST_EXIT_DBG(data.count(feSpaces[i]))("Should not happen!\n");
result += data.find(feSpaces[i])->second.rStartDofs[level];
result += data.find(feSpaces[i])->second.rStartDofs;
}
return result;
......@@ -304,19 +283,15 @@ namespace AMDiS {
it != feSpacesUnique.end(); ++it)
data[*it].update();
int nLevel = levelData->getLevelNumber();
for (int i = 0; i < nLevel; i++) {
// Compute all numbers from this mappings.
nRankDofs[i] = computeRankDofs(i);
nLocalDofs[i] = computeLocalDofs(i);
nOverallDofs[i] = computeOverallDofs(i);
rStartDofs[i] = computeStartDofs(i);
// And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal, i);
}
// Compute all numbers from this mappings.
nRankDofs = computeRankDofs();
nLocalDofs = computeLocalDofs();
nOverallDofs = computeOverallDofs();
rStartDofs = computeStartDofs();
// And finally, compute the matrix indices.
if (needMatIndex)
computeMatIndex(needMatIndexFromGlobal);
}
......@@ -335,16 +310,16 @@ namespace AMDiS {
}
void ParallelDofMapping::computeMatIndex(bool globalIndex, int level)
void ParallelDofMapping::computeMatIndex(bool globalIndex)
{
FUNCNAME("ParallelDofMapping::computeMatIndex()");
dofToMatIndex[level].clear();
dofToMatIndex.clear();
// The offset is always added to the local matrix index. The offset for the
// DOFs in the first FE spaces is the smalled global index of a DOF that is
// owned by the rank.
int offset = rStartDofs[level];
int offset = rStartDofs;
// === Create the matrix indices for all component FE spaces. ===
......@@ -353,20 +328,20 @@ namespace AMDiS {
// Traverse all DOFs of the FE space and create for all rank owned DOFs
// a matrix index.
DofMap& dofMap = data[feSpaces[i]].getMap(level);
DofMap& dofMap = data[feSpaces[i]].getMap();
for (DofMap::iterator it = dofMap.begin(); it != dofMap.end(); ++it) {
if (data[feSpaces[i]].isRankDof(it->first, level)) {
if (data[feSpaces[i]].isRankDof(it->first)) {
int globalMatIndex = it->second.local + offset;
if (globalIndex)
dofToMatIndex[level].add(i, it->second.global, globalMatIndex);
dofToMatIndex.add(i, it->second.global, globalMatIndex);
else
dofToMatIndex[level].add(i, it->first, globalMatIndex);
dofToMatIndex.add(i, it->first, globalMatIndex);
}
}
// Increase the offset for the next FE space by the number of DOFs owned
// by the rank in the current FE space.
offset += data[feSpaces[i]].nRankDofs[level];
offset += data[feSpaces[i]].nRankDofs;
// If there are no non local DOFs, continue with the next FE space.
if (!hasNonLocalDofs)
......@@ -378,16 +353,16 @@ namespace AMDiS {
// === interior boundaries. ===
StdMpi<vector<DegreeOfFreedom> > stdMpi(mpiComm);
for (DofComm::Iterator it(dofComm->getSendDofs(), level, feSpaces[i]);
for (DofComm::Iterator it(dofComm->getSendDofs(), 0, feSpaces[i]);
!it.end(); it.nextRank()) {
vector<DegreeOfFreedom> sendGlobalDofs;
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()))
if (globalIndex)
sendGlobalDofs.push_back(dofToMatIndex[level].get(i, dofMap[it.getDofIndex()].global));
sendGlobalDofs.push_back(dofToMatIndex.get(i, dofMap[it.getDofIndex()].global));
else
sendGlobalDofs.push_back(dofToMatIndex[level].get(i, it.getDofIndex()));
sendGlobalDofs.push_back(dofToMatIndex.get(i, it.getDofIndex()));
int rank = it.getRank();
if (meshLevel > 0)
......@@ -396,7 +371,7 @@ namespace AMDiS {
stdMpi.send(rank, sendGlobalDofs);
}
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpaces[i]);
for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]);
!it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
......@@ -408,7 +383,7 @@ namespace AMDiS {
stdMpi.startCommunication();
{
for (DofComm::Iterator it(dofComm->getRecvDofs(), level, feSpaces[i]);
for (DofComm::Iterator it(dofComm->getRecvDofs(), 0, feSpaces[i]);
!it.end(); it.nextRank()) {
int rank = it.getRank();
if (meshLevel > 0)
......@@ -419,9 +394,9 @@ namespace AMDiS {
if (dofMap.count(it.getDofIndex())) {
DegreeOfFreedom d = stdMpi.getRecvData(rank)[counter++];
if (globalIndex)
dofToMatIndex[level].add(i, dofMap[it.getDofIndex()].global, d);
dofToMatIndex.add(i, dofMap[it.getDofIndex()].global, d);
else
dofToMatIndex[level].add(i, it.getDofIndex(), d);
dofToMatIndex.add(i, it.getDofIndex(), d);
}
}
}
......
......@@ -107,7 +107,6 @@ namespace AMDiS {
: levelData(ld),
dofComm(NULL),
feSpace(NULL),
dofMap(1),
needGlobalMapping(false),
hasNonLocalDofs(false)
{
......@@ -117,75 +116,66 @@ namespace AMDiS {
/// Clears all data of the mapping.
void clear();
#if 0
/// Maps a DOF index to both, the local and global index of the mapping. The
/// global index must not be set.
MultiIndex& operator[](DegreeOfFreedom d)
{
TEST_EXIT_DBG(dofMap[0].count(d))("Should not happen!\n");
TEST_EXIT_DBG(dofMap.count(d))("Should not happen!\n");
return dofMap[0][d];
return dofMap[d];
}
#else
DofMap& operator[](int level)
{
TEST_EXIT_DBG(level < static_cast<int>(dofMap.size()))("Should not happen!\n");
return dofMap[level];
}
#endif
/// Inserts a new DOF to rank's mapping. The DOF is assumed to be owend by
/// the rank.
void insertRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{
FUNCNAME("FeSpaceDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap[level].count(dof0) == 0)("Should not happen!\n");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
dofMap[level][dof0].local = dof1;
nLocalDofs[level]++;
dofMap[dof0].local = dof1;
nLocalDofs++;
if (dof1 != -1)
nRankDofs[level]++;
nRankDofs++;
}
/// Inserts a new DOF to rank's mapping. The DOF exists in rank's subdomain
/// but is owned by a different rank, thus it is part of an interior boundary.
void insertNonRankDof(int level, DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
void insertNonRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{