Commit f1d2d34b authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on pdd.

parent 3d53fff8
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
#include "BoundaryManager.h" #include "BoundaryManager.h"
#include "Assembler.h" #include "Assembler.h"
#include "mpi.h"
namespace AMDiS { namespace AMDiS {
using namespace mtl; using namespace mtl;
...@@ -222,17 +224,6 @@ namespace AMDiS { ...@@ -222,17 +224,6 @@ namespace AMDiS {
DegreeOfFreedom col = colIndices[j]; DegreeOfFreedom col = colIndices[j];
double entry = elMat[i][j]; double entry = elMat[i][j];
/*
if (MPI::COMM_WORLD.Get_rank() == 0 && (row >= 156 || col >= 156))
std::cout << "PROB 0: " << row << " " << col << std::endl;
if (MPI::COMM_WORLD.Get_rank() == 1 && (row >= 151 || col >= 151))
std::cout << "PROB 1: " << row << " " << col << std::endl;
if (MPI::COMM_WORLD.Get_rank() == 2 && (row >= 146 || col >= 146))
std::cout << "PROB 2: " << row << " " << col << std::endl;
if (MPI::COMM_WORLD.Get_rank() == 3 && (row >= 153 || col >= 153))
std::cout << "PROB 3: " << row << " " << col << std::endl;
*/
if (add) if (add)
ins[row][col] += sign * entry; ins[row][col] += sign * entry;
else else
......
...@@ -83,4 +83,5 @@ namespace AMDiS { ...@@ -83,4 +83,5 @@ namespace AMDiS {
return true; return true;
} }
} }
...@@ -66,6 +66,19 @@ namespace AMDiS { ...@@ -66,6 +66,19 @@ namespace AMDiS {
return &dofs[node0 + elementPos][n0 + dofPos]; return &dofs[node0 + elementPos][n0 + dofPos];
} }
/// Returns \ref pos, the current position (vertex, edge, face) of the traverse.
int getCurrentPos()
{
return pos;
}
/// Returns \ref elementPos, the number of vertex, edge or face that is traversed.
int getCurrentElementPos()
{
return elementPos;
}
protected: protected:
/// The dof admin for which dofs should be traversed. /// The dof admin for which dofs should be traversed.
const DOFAdmin* admin; const DOFAdmin* admin;
...@@ -108,7 +121,7 @@ namespace AMDiS { ...@@ -108,7 +121,7 @@ namespace AMDiS {
*/ */
int nElements; int nElements;
/// Current element (vertex, edge, face) that is traversed. /// Current element, i.e., ith vertex, edge or face, that is traversed.
int elementPos; int elementPos;
/// Currrent dof that is traversed on the current element; /// Currrent dof that is traversed on the current element;
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include "AdaptInstationary.h" #include "AdaptInstationary.h"
#include "FiniteElemSpace.h" #include "FiniteElemSpace.h"
#include "ElementData.h" #include "ElementData.h"
#include "ElementDofIterator.h"
#include "MacroElement.h" #include "MacroElement.h"
#include "MacroReader.h" #include "MacroReader.h"
#include "Mesh.h" #include "Mesh.h"
...@@ -867,18 +868,22 @@ namespace AMDiS { ...@@ -867,18 +868,22 @@ namespace AMDiS {
{ {
DimVec<double>* baryCoords; DimVec<double>* baryCoords;
bool found = false; bool found = false;
ElementDofIterator elDofIter(feSpace->getAdmin());
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, ElInfo *elInfo = stack.traverseFirst(this, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS); Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) { while (elInfo) {
for (int i = 0; i < nDOFEl; i++) { elDofIter.reset(elInfo->getElement());
if (elInfo->getElement()->getDOF(i) == dof) { int i = 0;
do {
if (elDofIter.getDofPtr() == dof) {
baryCoords = feSpace->getBasisFcts()->getCoords(i); baryCoords = feSpace->getBasisFcts()->getCoords(i);
elInfo->coordToWorld(*baryCoords, coords); elInfo->coordToWorld(*baryCoords, coords);
found = true; found = true;
break; break;
} }
} i++;
} while (elDofIter.next());
if (found) if (found)
break; break;
......
...@@ -38,6 +38,13 @@ namespace AMDiS { ...@@ -38,6 +38,13 @@ namespace AMDiS {
initialPartitionMesh(true), initialPartitionMesh(true),
nRankDOFs(0) nRankDOFs(0)
{ {
FUNCNAME("ParallelDomainBase::ParalleDomainBase()");
TEST_EXIT(mesh->getNumberOfDOFAdmin() == 1)
("Only meshes with one DOFAdmin are supported!\n");
TEST_EXIT(mesh->getDOFAdmin(0).getNumberOfPreDOFs(0) == 0)
("Wrong pre dof number for DOFAdmin!\n");
mpiRank = MPI::COMM_WORLD.Get_rank(); mpiRank = MPI::COMM_WORLD.Get_rank();
mpiSize = MPI::COMM_WORLD.Get_size(); mpiSize = MPI::COMM_WORLD.Get_size();
mpiComm = MPI::COMM_WORLD; mpiComm = MPI::COMM_WORLD;
...@@ -100,8 +107,6 @@ namespace AMDiS { ...@@ -100,8 +107,6 @@ namespace AMDiS {
admin.setFirstHole(mapLocalGlobalDOFs.size()); admin.setFirstHole(mapLocalGlobalDOFs.size());
} }
exit(0);
// === Global refinements. === // === Global refinements. ===
int globalRefinement = 0; int globalRefinement = 0;
...@@ -114,9 +119,9 @@ namespace AMDiS { ...@@ -114,9 +119,9 @@ namespace AMDiS {
} }
// #if (DEBUG != 0) #if (DEBUG != 0)
// testInteriorBoundary(); testInteriorBoundary();
// #endif #endif
// === Create petsc matrix. === // === Create petsc matrix. ===
...@@ -234,8 +239,7 @@ namespace AMDiS { ...@@ -234,8 +239,7 @@ namespace AMDiS {
int requestCounter = 0; int requestCounter = 0;
int i = 0; int i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator sendIt = sendDofs.begin();
sendIt = sendDofs.begin();
sendIt != sendDofs.end(); sendIt != sendDofs.end();
++sendIt, i++) { ++sendIt, i++) {
int nSendDOFs = sendIt->second.size(); int nSendDOFs = sendIt->second.size();
...@@ -249,8 +253,7 @@ namespace AMDiS { ...@@ -249,8 +253,7 @@ namespace AMDiS {
} }
i = 0; i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt = recvDofs.begin();
recvIt != recvDofs.end(); recvIt != recvDofs.end();
++recvIt, i++) { ++recvIt, i++) {
int nRecvDOFs = recvIt->second.size(); int nRecvDOFs = recvIt->second.size();
...@@ -265,12 +268,12 @@ namespace AMDiS { ...@@ -265,12 +268,12 @@ namespace AMDiS {
i = 0; i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator recvIt = recvDofs.begin();
recvIt = recvDofs.begin();
recvIt != recvDofs.end(); recvIt != recvDofs.end();
++recvIt, i++) { ++recvIt, i++) {
for (int j = 0; j < static_cast<int>(recvIt->second.size()); j++) for (int j = 0; j < static_cast<int>(recvIt->second.size()); j++) {
(*vec)[(recvIt->second)[j][0]] = recvBuffers[i][j]; (*vec)[*(recvIt->second)[j]] = recvBuffers[i][j];
}
delete [] recvBuffers[i]; delete [] recvBuffers[i];
} }
...@@ -330,8 +333,8 @@ namespace AMDiS { ...@@ -330,8 +333,8 @@ namespace AMDiS {
} }
void ParallelDomainBase::createInteriorBoundaryInfo(std::vector<const DegreeOfFreedom*>& rankDOFs, void ParallelDomainBase::createInteriorBoundaryInfo(DofContainer& rankDOFs,
std::map<const DegreeOfFreedom*, int>& boundaryDOFs) DofToRank& boundaryDOFs)
{ {
FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()"); FUNCNAME("ParallelDomainBase::createInteriorBoundaryInfo()");
...@@ -499,14 +502,14 @@ namespace AMDiS { ...@@ -499,14 +502,14 @@ namespace AMDiS {
} }
void ParallelDomainBase::createLocalGlobalNumbering( void ParallelDomainBase::createLocalGlobalNumbering(DofContainer& rankDOFs,
std::vector<const DegreeOfFreedom*>& rankDOFs, DofToRank& boundaryDOFs,
std::map<const DegreeOfFreedom*, int>& boundaryDOFs,
int& nRankDOFs, int& nRankDOFs,
int& nOverallDOFs) int& nOverallDOFs)
{ {
FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()"); FUNCNAME("ParallelDomainBase::createLocalGlobalNumbering()");
// === Get rank information about DOFs. === // === Get rank information about DOFs. ===
// Stores to each DOF pointer the set of ranks the DOF is part of. // Stores to each DOF pointer the set of ranks the DOF is part of.
...@@ -537,7 +540,7 @@ namespace AMDiS { ...@@ -537,7 +540,7 @@ namespace AMDiS {
// another rank. // another rank.
std::map<int, int> recvNewDofs; std::map<int, int> recvNewDofs;
for (std::map<const DegreeOfFreedom*, int>::iterator it = boundaryDOFs.begin(); for (DofToRank::iterator it = boundaryDOFs.begin();
it != boundaryDOFs.end(); it != boundaryDOFs.end();
++it) { ++it) {
...@@ -570,7 +573,6 @@ namespace AMDiS { ...@@ -570,7 +573,6 @@ namespace AMDiS {
} }
} }
// === Send and receive the dof indices at boundary. === // === Send and receive the dof indices at boundary. ===
std::vector<int*> sendBuffers(sendNewDofs.size()); std::vector<int*> sendBuffers(sendNewDofs.size());
...@@ -646,6 +648,20 @@ namespace AMDiS { ...@@ -646,6 +648,20 @@ namespace AMDiS {
// === Change dof indices at boundary from other ranks. === // === Change dof indices at boundary from other ranks. ===
// Within this small data structure we track which dof index was already changed.
// This is used to avoid the following situation: Assume, there are two dof indices
// a and b in boundaryDOFs. Then we have to change index a to b and b to c. When
// the second rule applies, we have to avoid that not the first b, resulted from
// changing a to b, is set to c, but the second one. Therefore, after the first
// rule was applied, the dof pointer is set to false in this data structure and
// is not allowed to be changed anymore.
std::map<const DegreeOfFreedom*, bool> dofChanged;
for (DofToRank::iterator dofIt = boundaryDOFs.begin();
dofIt != boundaryDOFs.end();
++dofIt) {
dofChanged[dofIt->first] = false;
}
i = 0; i = 0;
for (std::map<int, int>::iterator recvIt = recvNewDofs.begin(); for (std::map<int, int>::iterator recvIt = recvNewDofs.begin();
recvIt != recvNewDofs.end(); recvIt != recvNewDofs.end();
...@@ -659,13 +675,18 @@ namespace AMDiS { ...@@ -659,13 +675,18 @@ namespace AMDiS {
bool found = false; bool found = false;
for (std::map<const DegreeOfFreedom*, int>::iterator dofIt = boundaryDOFs.begin(); // Iterate over all boundary dofs to find the dof, which index we have to change.
for (DofToRank::iterator dofIt = boundaryDOFs.begin();
dofIt != boundaryDOFs.end(); dofIt != boundaryDOFs.end();
++dofIt) { ++dofIt) {
if ((dofIt->first)[0] == oldDof) { if (*(dofIt->first) == oldDof && !dofChanged[dofIt->first]) {
dofChanged[dofIt->first] = true;
recvDofs[recvIt->first].push_back(dofIt->first); recvDofs[recvIt->first].push_back(dofIt->first);
const_cast<DegreeOfFreedom*>(dofIt->first)[0] = newLocalDof; *(const_cast<DegreeOfFreedom*>(dofIt->first)) = newLocalDof;
mapLocalGlobalDOFs[newLocalDof] = newGlobalDof; mapLocalGlobalDOFs[newLocalDof] = newGlobalDof;
mapGlobalLocalDOFs[newGlobalDof] = newLocalDof; mapGlobalLocalDOFs[newGlobalDof] = newLocalDof;
isRankDOF[newLocalDof] = false; isRankDOF[newLocalDof] = false;
...@@ -688,27 +709,31 @@ namespace AMDiS { ...@@ -688,27 +709,31 @@ namespace AMDiS {
FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()"); FUNCNAME("ParallelDomainBase::updateLocalGlobalNumbering()");
std::set<const DegreeOfFreedom*> rankDOFs; std::set<const DegreeOfFreedom*> rankDOFs;
std::map<const DegreeOfFreedom*, int> newBoundaryDOFs; DofToRank newBoundaryDOFs;
// === Get all DOFs in ranks partition. === // === Get all DOFs in ranks partition. ===
ElementDofIterator elDofIt(feSpace->getAdmin());
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL); ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) { while (elInfo) {
Element *element = elInfo->getElement(); Element *element = elInfo->getElement();
for (int i = 0; i < 3; i++) elDofIt.reset(element);
rankDOFs.insert(element->getDOF(i)); do {
rankDOFs.insert(elDofIt.getDofPtr());
} while(elDofIt.next());
elInfo = stack.traverseNext(elInfo); elInfo = stack.traverseNext(elInfo);
} }
// === Traverse on interior boundaries and move all not ranked owned DOFs from === // === Traverse on interior boundaries and move all not ranked owned DOFs from ===
// === rankDOFs to boundaryDOFs. === // === rankDOFs to boundaryDOFs. ===
std::map<int, std::vector<const DegreeOfFreedom*> > sendNewDofs = sendDofs; RankToDofContainer sendNewDofs;
std::map<int, std::vector<const DegreeOfFreedom*> > recvNewDofs = recvDofs; RankToDofContainer recvNewDofs;
for (std::map<int, std::vector<AtomicBoundary> >::iterator it = for (std::map<int, std::vector<AtomicBoundary> >::iterator it =
myIntBoundary.boundary.begin(); myIntBoundary.boundary.begin();
...@@ -745,9 +770,21 @@ namespace AMDiS { ...@@ -745,9 +770,21 @@ namespace AMDiS {
newBoundaryDOFs[dof1] = boundaryDOFs[dof1]; newBoundaryDOFs[dof1] = boundaryDOFs[dof1];
newBoundaryDOFs[dof2] = boundaryDOFs[dof2]; newBoundaryDOFs[dof2] = boundaryDOFs[dof2];
std::vector<const DegreeOfFreedom*> boundDOFs; if (find(sendNewDofs[it->first].begin(), sendNewDofs[it->first].end(), dof1) ==
addAllDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary, sendNewDofs[it->first].end())
sendNewDofs[it->first].push_back(dof1);
if (find(sendNewDofs[it->first].begin(), sendNewDofs[it->first].end(), dof2) ==
sendNewDofs[it->first].end())
sendNewDofs[it->first].push_back(dof2);
DofContainer boundDOFs;
addAllVertexDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
addAllEdgeDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs); boundDOFs);
for (int i = 0; i < static_cast<int>(boundDOFs.size()); i++) { for (int i = 0; i < static_cast<int>(boundDOFs.size()); i++) {
newBoundaryDOFs[boundDOFs[i]] = mpiRank; newBoundaryDOFs[boundDOFs[i]] = mpiRank;
sendNewDofs[it->first].push_back(boundDOFs[i]); sendNewDofs[it->first].push_back(boundDOFs[i]);
...@@ -755,7 +792,6 @@ namespace AMDiS { ...@@ -755,7 +792,6 @@ namespace AMDiS {
} }
} }
for (std::map<int, std::vector<AtomicBoundary> >::iterator it = for (std::map<int, std::vector<AtomicBoundary> >::iterator it =
otherIntBoundary.boundary.begin(); otherIntBoundary.boundary.begin();
it != otherIntBoundary.boundary.end(); it != otherIntBoundary.boundary.end();
...@@ -793,9 +829,21 @@ namespace AMDiS { ...@@ -793,9 +829,21 @@ namespace AMDiS {
newBoundaryDOFs[dof1] = boundaryDOFs[dof1]; newBoundaryDOFs[dof1] = boundaryDOFs[dof1];
newBoundaryDOFs[dof2] = boundaryDOFs[dof2]; newBoundaryDOFs[dof2] = boundaryDOFs[dof2];
std::vector<const DegreeOfFreedom*> boundDOFs; if (find(recvNewDofs[it->first].begin(), recvNewDofs[it->first].end(), dof2) ==
addAllDOFs(boundIt->rankObject.el, boundIt->rankObject.ithObjAtBoundary, recvNewDofs[it->first].end())
recvNewDofs[it->first].push_back(dof2);
if (find(recvNewDofs[it->first].begin(), recvNewDofs[it->first].end(), dof1) ==
recvNewDofs[it->first].end())
recvNewDofs[it->first].push_back(dof1);
DofContainer boundDOFs;
addAllEdgeDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs);
addAllVertexDOFs(boundIt->rankObject.el,
boundIt->rankObject.ithObjAtBoundary,
boundDOFs); boundDOFs);
for (int i = static_cast<int>(boundDOFs.size()) - 1; i >= 0; i--) { for (int i = static_cast<int>(boundDOFs.size()) - 1; i >= 0; i--) {
// for (int i = 0; i < static_cast<int>(boundDOFs.size()); i++) { // for (int i = 0; i < static_cast<int>(boundDOFs.size()); i++) {
rankDOFs.erase(boundDOFs[i]); rankDOFs.erase(boundDOFs[i]);
...@@ -803,7 +851,6 @@ namespace AMDiS { ...@@ -803,7 +851,6 @@ namespace AMDiS {
recvNewDofs[it->first].push_back(boundDOFs[i]); recvNewDofs[it->first].push_back(boundDOFs[i]);
} }
} }
} }
nRankDOFs = rankDOFs.size(); nRankDOFs = rankDOFs.size();
...@@ -846,14 +893,13 @@ namespace AMDiS { ...@@ -846,14 +893,13 @@ namespace AMDiS {
int requestCounter = 0; int requestCounter = 0;
i = 0; i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator sendIt = sendNewDofs.begin();
sendIt = sendNewDofs.begin();
sendIt != sendNewDofs.end(); sendIt != sendNewDofs.end();
++sendIt, i++) { ++sendIt, i++) {
int nSendDofs = sendIt->second.size(); int nSendDofs = sendIt->second.size();
sendBuffers[i] = new int[nSendDofs]; sendBuffers[i] = new int[nSendDofs];
int c = 0; int c = 0;
for (std::vector<const DegreeOfFreedom*>::iterator dofIt = sendIt->second.begin(); for (DofContainer::iterator dofIt = sendIt->second.begin();
dofIt != sendIt->second.end(); dofIt != sendIt->second.end();
++dofIt) ++dofIt)
sendBuffers[i][c++] = (*dofIt)[0] + rstart; sendBuffers[i][c++] = (*dofIt)[0] + rstart;
...@@ -863,9 +909,9 @@ namespace AMDiS { ...@@ -863,9 +909,9 @@ namespace AMDiS {
} }
i = 0; i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator recvIt = recvNewDofs.begin();
recvIt = recvNewDofs.begin(); recvIt != recvNewDofs.end();
recvIt != recvNewDofs.end(); ++recvIt, i++) { ++recvIt, i++) {
int nRecvDofs = recvIt->second.size(); int nRecvDofs = recvIt->second.size();
recvBuffers[i] = new int[nRecvDofs]; recvBuffers[i] = new int[nRecvDofs];
...@@ -873,28 +919,25 @@ namespace AMDiS { ...@@ -873,28 +919,25 @@ namespace AMDiS {
mpiComm.Irecv(recvBuffers[i], nRecvDofs, MPI_INT, recvIt->first, 0); mpiComm.Irecv(recvBuffers[i], nRecvDofs, MPI_INT, recvIt->first, 0);
} }
MPI::Request::Waitall(requestCounter, request); MPI::Request::Waitall(requestCounter, request);
i = 0; i = 0;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator sendIt = sendNewDofs.begin();
sendIt = sendNewDofs.begin();
sendIt != sendNewDofs.end(); sendIt != sendNewDofs.end();
++sendIt) ++sendIt)
delete [] sendBuffers[i++]; delete [] sendBuffers[i++];
i = 0; i = 0;
int newDofIndex = nRankDOFs; int newDofIndex = nRankDOFs;
for (std::map<int, std::vector<const DegreeOfFreedom*> >::iterator for (RankToDofContainer::iterator recvIt = recvNewDofs.begin();
recvIt = recvNewDofs.begin(); recvIt != recvNewDofs.end();
recvIt != recvNewDofs.end(); ++recvIt) { ++recvIt) {
int j = 0; int j = 0;
for (std::vector<const DegreeOfFreedom*>::iterator dofIt = recvIt->second.begin(); for (DofContainer::iterator dofIt = recvIt->second.begin();
dofIt != recvIt->second.end(); dofIt != recvIt->second.end();
++dofIt) { ++dofIt) {
const_cast<DegreeOfFreedom*>(*dofIt)[0] = newDofIndex; (*const_cast<DegreeOfFreedom*>(*dofIt)) = newDofIndex;
mapLocalGlobalDOFs[newDofIndex] = recvBuffers[i][j]; mapLocalGlobalDOFs[newDofIndex] = recvBuffers[i][j];
mapGlobalLocalDOFs[recvBuffers[i][j]] = newDofIndex; mapGlobalLocalDOFs[recvBuffers[i][j]] = newDofIndex;
isRankDOF[newDofIndex] = false; isRankDOF[newDofIndex] = false;
...@@ -912,31 +955,31 @@ namespace AMDiS { ...@@ -912,31 +955,31 @@ namespace AMDiS {
} }
void ParallelDomainBase::addAllDOFs(Element *el, int ithEdge, void ParallelDomainBase::addAllVertexDOFs(Element *el, int ithEdge,
std::vector<const DegreeOfFreedom*>& dofs) DofContainer& dofs)
{ {
FUNCNAME("ParallelDomainBase::addAllDOFs()"); FUNCNAME("ParallelDomainBase::addAllVertexDOFs()");
switch (ithEdge) { switch (ithEdge) {
case 0: case 0:
if (el->getSecondChild() && el->getSecondChild()->getFirstChild()) { if (el->getSecondChild() && el->getSecondChild()->getFirstChild()) {