Liebe Gitlab-Nutzerin, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 7cb626f3 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

FETI-DP

parent 526e4fd2
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
#include "parallel/FeSpaceMapping.h" #include "parallel/FeSpaceMapping.h"
#include "parallel/StdMpi.h"
namespace AMDiS { namespace AMDiS {
...@@ -28,7 +29,7 @@ namespace AMDiS { ...@@ -28,7 +29,7 @@ namespace AMDiS {
void GlobalDofMap::update(bool add) void GlobalDofMap::update(bool add)
{ {
for (DofMapping::iterator it = dofMap.begin(); it != dofMap.end(); ++it) for (DofMapping::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (it->second == -1) if (it->second == -1 && nonRankDofs.count(it->first) == 0)
it->second = nRankDofs++; it->second = nRankDofs++;
nOverallDofs = 0; nOverallDofs = 0;
...@@ -37,6 +38,9 @@ namespace AMDiS { ...@@ -37,6 +38,9 @@ namespace AMDiS {
if (add) if (add)
addOffset(rStartDofs); addOffset(rStartDofs);
if (overlap)
computeNonLocalIndices();
} }
...@@ -46,4 +50,55 @@ namespace AMDiS { ...@@ -46,4 +50,55 @@ namespace AMDiS {
it->second += offset; it->second += offset;
} }
void GlobalDofMap::computeNonLocalIndices()
{
FUNCNAME("GlobalDofMap::computeNonLocalIndices()");
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;
StdMpi<vector<int> > stdMpi(*mpiComm);
for (DofComm::Iterator it(*sendDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
if (dofMap.count(it.getDofIndex()) && !nonRankDofs.count(it.getDofIndex()))
stdMpi.getSendData(it.getRank()).push_back(dofMap[it.getDofIndex()]);
stdMpi.updateSendDataSize();
for (DofComm::Iterator it(*recvDofs, feSpace); !it.end(); it.nextRank()) {
bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) {
if (nonRankDofs.count(it.getDofIndex())) {
recvFromRank = true;
break;
}
}
if (recvFromRank)
stdMpi.recv(it.getRank());
}
stdMpi.startCommunication();
for (DofComm::Iterator it(*recvDofs, feSpace);
!it.end(); it.nextRank()) {
int i = 0;
for (; !it.endDofIter(); it.nextDof())
if (nonRankDofs.count(it.getDofIndex()))
dofMap[it.getDofIndex()] = stdMpi.getRecvData(it.getRank())[i++];
}
}
void GlobalDofMap::print()
{
FUNCNAME("GlobalDofMap::print()");
MSG("Local to global mapping on this rank: \n");
for (DofMapping::iterator it = dofMap.begin(); it != dofMap.end(); ++it)
if (nonRankDofs.count(it->first) == 0)
MSG(" %d -> %d (rank-dof)\n", it->first, it->second);
else
MSG(" %d -> %d \n", it->first, it->second);
}
} }
...@@ -22,6 +22,8 @@ ...@@ -22,6 +22,8 @@
#include <vector> #include <vector>
#include <map> #include <map>
#include <set>
#include "parallel/DofComm.h"
#include "parallel/MpiHelper.h" #include "parallel/MpiHelper.h"
#include "parallel/ParallelTypes.h" #include "parallel/ParallelTypes.h"
...@@ -43,9 +45,11 @@ namespace AMDiS { ...@@ -43,9 +45,11 @@ namespace AMDiS {
GlobalDofMap(MPI::Intracomm* m) GlobalDofMap(MPI::Intracomm* m)
: mpiComm(m), : mpiComm(m),
feSpace(NULL),
nRankDofs(0), nRankDofs(0),
nOverallDofs(0), nOverallDofs(0),
rStartDofs(0) rStartDofs(0),
overlap(false)
{} {}
void clear(); void clear();
...@@ -57,22 +61,24 @@ namespace AMDiS { ...@@ -57,22 +61,24 @@ namespace AMDiS {
return dofMap[d]; return dofMap[d];
} }
void insertRankDof(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1) void insertRankDof(DegreeOfFreedom dof0)
{ {
FUNCNAME("GlobalDofMap::insertRankDof()"); FUNCNAME("GlobalDofMap::insertRankDof()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
dofMap[dof0] = dof1; dofMap[dof0] = -1;
} }
void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1) void insert(DegreeOfFreedom dof0, DegreeOfFreedom dof1 = -1)
{ {
FUNCNAME("GlobalDofMap::insert()"); FUNCNAME("GlobalDofMap::insert()");
TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n"); TEST_EXIT_DBG(dofMap.count(dof0) == 0)("Should not happen!\n");
dofMap[dof0] = dof1; dofMap[dof0] = dof1;
nonRankDofs.insert(dof0);
} }
bool isSet(DegreeOfFreedom dof) bool isSet(DegreeOfFreedom dof)
...@@ -94,15 +100,44 @@ namespace AMDiS { ...@@ -94,15 +100,44 @@ namespace AMDiS {
void addOffset(int offset); void addOffset(int offset);
void computeNonLocalIndices();
void print();
void setFeSpace(const FiniteElemSpace *fe)
{
feSpace = fe;
}
void setOverlap(bool b)
{
overlap = b;
}
void setDofComm(DofComm &pSend, DofComm &pRecv)
{
sendDofs = &pSend;
recvDofs = &pRecv;
}
private: private:
MPI::Intracomm* mpiComm; MPI::Intracomm* mpiComm;
const FiniteElemSpace *feSpace;
/// ///
DofMapping dofMap; DofMapping dofMap;
std::set<DegreeOfFreedom> nonRankDofs;
DofComm *sendDofs;
DofComm *recvDofs;
public: public:
/// ///
int nRankDofs, nOverallDofs, rStartDofs; int nRankDofs, nOverallDofs, rStartDofs;
bool overlap;
}; };
...@@ -140,6 +175,8 @@ namespace AMDiS { ...@@ -140,6 +175,8 @@ namespace AMDiS {
data.find(feSpace)->second.clear(); data.find(feSpace)->second.clear();
else else
data.insert(make_pair(feSpace, T(mpiComm))); data.insert(make_pair(feSpace, T(mpiComm)));
data.find(feSpace)->second.setFeSpace(feSpace);
} }
int getRankDofs(vector<const FiniteElemSpace*> &fe) int getRankDofs(vector<const FiniteElemSpace*> &fe)
...@@ -203,7 +240,6 @@ namespace AMDiS { ...@@ -203,7 +240,6 @@ namespace AMDiS {
feSpaces = fe; feSpaces = fe;
for (unsigned int i = 0; i < feSpaces.size(); i++) for (unsigned int i = 0; i < feSpaces.size(); i++)
addFeSpace(feSpaces[i]); addFeSpace(feSpaces[i]);
} }
void update() void update()
......
...@@ -251,60 +251,18 @@ namespace AMDiS { ...@@ -251,60 +251,18 @@ namespace AMDiS {
for (DofIndexSet::iterator it = primals.begin(); it != primals.end(); ++it) for (DofIndexSet::iterator it = primals.begin(); it != primals.end(); ++it)
if (meshDistributor->getIsRankDof(feSpace, *it)) if (meshDistributor->getIsRankDof(feSpace, *it))
primalDofMap[feSpace].insertRankDof(*it); primalDofMap[feSpace].insertRankDof(*it);
else
primalDofMap[feSpace].insert(*it);
primalDofMap[feSpace].setOverlap(true);
primalDofMap[feSpace].setDofComm(meshDistributor->getSendDofs(),
meshDistributor->getRecvDofs());
primalDofMap[feSpace].update(); primalDofMap[feSpace].update();
MSG("nRankPrimals = %d nOverallPrimals = %d\n", MSG("nRankPrimals = %d nOverallPrimals = %d\n",
primalDofMap[feSpace].nRankDofs, primalDofMap[feSpace].nRankDofs,
primalDofMap[feSpace].nOverallDofs); primalDofMap[feSpace].nOverallDofs);
// === Communicate primal's global index from ranks that own the ===
// === primals to ranks that contain this primals but are not owning ===
// === them. ===
typedef map<int, map<const FiniteElemSpace*, DofContainer> >::iterator it_type;
StdMpi<vector<int> > stdMpi(meshDistributor->getMpiComm());
for (DofComm::Iterator it(meshDistributor->getSendDofs(), feSpace);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
if (primalDofMap[feSpace].isSet(it.getDofIndex())) {
DegreeOfFreedom d = primalDofMap[feSpace][it.getDofIndex()];
stdMpi.getSendData(it.getRank()).push_back(d);
}
stdMpi.updateSendDataSize();
for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
!it.end(); it.nextRank()) {
bool recvFromRank = false;
for (; !it.endDofIter(); it.nextDof()) {
if (primals.count(it.getDofIndex()) &&
meshDistributor->getIsRankDof(feSpace, it.getDofIndex()) == false) {
recvFromRank = true;
break;
}
}
if (recvFromRank)
stdMpi.recv(it.getRank());
}
stdMpi.startCommunication();
for (DofComm::Iterator it(meshDistributor->getRecvDofs(), feSpace);
!it.end(); it.nextRank()) {
int i = 0;
for (; !it.endDofIter(); it.nextDof()) {
if (primals.count(it.getDofIndex()) &&
meshDistributor->getIsRankDof(feSpace, it.getDofIndex()) == false) {
DegreeOfFreedom d = stdMpi.getRecvData(it.getRank())[i++];
primalDofMap[feSpace].insert(it.getDofIndex(), d);
}
}
}
TEST_EXIT_DBG(primals.size() == primalDofMap[feSpace].size()) TEST_EXIT_DBG(primals.size() == primalDofMap[feSpace].size())
("Number of primals %d, but number of global primals on this rank is %d!\n", ("Number of primals %d, but number of global primals on this rank is %d!\n",
primals.size(), primalDofMap[feSpace].size()); primals.size(), primalDofMap[feSpace].size());
...@@ -472,7 +430,7 @@ namespace AMDiS { ...@@ -472,7 +430,7 @@ namespace AMDiS {
nLocalInterior++; nLocalInterior++;
} }
} }
localDofMap[feSpace].update(false); localDofMap[feSpace].update(false);
TEST_EXIT_DBG(nLocalInterior + TEST_EXIT_DBG(nLocalInterior +
...@@ -919,13 +877,7 @@ namespace AMDiS { ...@@ -919,13 +877,7 @@ namespace AMDiS {
for (DofMapping::iterator it = localDofMap[feSpace].getMap().begin(); for (DofMapping::iterator it = localDofMap[feSpace].getMap().begin();
it != localDofMap[feSpace].getMap().end(); ++it) { it != localDofMap[feSpace].getMap().end(); ++it) {
#if 1
int petscIndex = localDofMap.mapLocal(it->first, i); int petscIndex = localDofMap.mapLocal(it->first, i);
#else
int petscIndex = it->second * nComponents + i;
#endif
dofVec[it->first] = localSolB[petscIndex]; dofVec[it->first] = localSolB[petscIndex];
} }
...@@ -1106,12 +1058,7 @@ namespace AMDiS { ...@@ -1106,12 +1058,7 @@ namespace AMDiS {
if (rowIndex < nLocalInterior) { if (rowIndex < nLocalInterior) {
if (colIndex < nLocalInterior) { if (colIndex < nLocalInterior) {
#if 1
int colIndex2 = localDofMap.mapLocal(col(*icursor), j); int colIndex2 = localDofMap.mapLocal(col(*icursor), j);
#else
int colIndex2 = localDofMap[feSpace][col(*icursor)] * nComponents + j;
#endif
colsLocal.push_back(colIndex2); colsLocal.push_back(colIndex2);
valuesLocal.push_back(value(*icursor)); valuesLocal.push_back(value(*icursor));
} else { } else {
...@@ -1123,12 +1070,7 @@ namespace AMDiS { ...@@ -1123,12 +1070,7 @@ namespace AMDiS {
} }
} else { } else {
if (colIndex < nLocalInterior) { if (colIndex < nLocalInterior) {
#if 1
int colIndex2 = localDofMap.mapLocal(col(*icursor), j); int colIndex2 = localDofMap.mapLocal(col(*icursor), j);
#else
int colIndex2 = localDofMap[feSpace][col(*icursor)] * nComponents + j;
#endif
colsLocalOther.push_back(colIndex2); colsLocalOther.push_back(colIndex2);
valuesLocalOther.push_back(value(*icursor)); valuesLocalOther.push_back(value(*icursor));
} else { } else {
...@@ -1159,41 +1101,22 @@ namespace AMDiS { ...@@ -1159,41 +1101,22 @@ namespace AMDiS {
if (colsOther.size()) { if (colsOther.size()) {
for (unsigned int k = 0; k < colsOther.size(); k++) for (unsigned int k = 0; k < colsOther.size(); k++)
#if 1
colsOther[k] = localDofMap.mapGlobal(colsOther[k], j); colsOther[k] = localDofMap.mapGlobal(colsOther[k], j);
#else
colsOther[k] = (localDofMap[feSpace][colsOther[k]] + MatSetValues(mat_primal_b, 1, &rowIndex, colsOther.size(),
localDofMap[feSpace].rStartDofs) * nComponents + j;
#endif
MatSetValues(mat_primal_b, 1, &rowIndex, colsOther.size(),
&(colsOther[0]), &(valuesOther[0]), ADD_VALUES); &(colsOther[0]), &(valuesOther[0]), ADD_VALUES);
} }
} else { } else {
#if 1
int localRowIndex = localDofMap.mapLocal(*cursor, i); int localRowIndex = localDofMap.mapLocal(*cursor, i);
#else
int localRowIndex = localDofMap[feSpace][*cursor] * nComponents + i;
#endif
for (unsigned int k = 0; k < cols.size(); k++) for (unsigned int k = 0; k < cols.size(); k++)
#if 1
cols[k] = localDofMap.mapLocal(cols[k], j); cols[k] = localDofMap.mapLocal(cols[k], j);
#else
cols[k] = localDofMap[feSpace][cols[k]] * nComponents + j;
#endif
MatSetValues(mat_b_b, 1, &localRowIndex, cols.size(), MatSetValues(mat_b_b, 1, &localRowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES); &(cols[0]), &(values[0]), ADD_VALUES);
if (colsOther.size()) { if (colsOther.size()) {
#if 1
int globalRowIndex = localDofMap.mapGlobal(*cursor, i); int globalRowIndex = localDofMap.mapGlobal(*cursor, i);
#else
int globalRowIndex =
(localDofMap[feSpace][*cursor] + localDofMap[feSpace].rStartDofs) * nComponents + i;
#endif
for (unsigned int k = 0; k < colsOther.size(); k++) for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] = colsOther[k] =
...@@ -1338,12 +1261,7 @@ namespace AMDiS { ...@@ -1338,12 +1261,7 @@ namespace AMDiS {
double value = *dofIt; double value = *dofIt;
VecSetValues(f_primal, 1, &index, &value, ADD_VALUES); VecSetValues(f_primal, 1, &index, &value, ADD_VALUES);
} else { } else {
#if 1
index = localDofMap.mapGlobal(index, i); index = localDofMap.mapGlobal(index, i);
#else
index = (localDofMap[feSpace][index] + localDofMap[feSpace].rStartDofs) * nComponents + i;
#endif
VecSetValue(f_b, index, *dofIt, INSERT_VALUES); VecSetValue(f_b, index, *dofIt, INSERT_VALUES);
} }
} }
......
...@@ -712,9 +712,6 @@ namespace AMDiS { ...@@ -712,9 +712,6 @@ namespace AMDiS {
int globalMatIndex = int globalMatIndex =
globalIndex - meshDistributor->getStartDofs(feSpaces[i]) + offset; globalIndex - meshDistributor->getStartDofs(feSpaces[i]) + offset;
// int globalMatIndex =
// globalIndex * feSpaces.size() + i;
dofToMatIndex.add(i, globalIndex, globalMatIndex); dofToMatIndex.add(i, globalIndex, globalMatIndex);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment