Commit a24ca7a5 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed several bugs in parallel code. Added more parallel check and debug functions.

parent f9e40e6c
...@@ -178,6 +178,21 @@ namespace AMDiS { ...@@ -178,6 +178,21 @@ namespace AMDiS {
} }
Element* getLevel0ParentElement(Mesh *mesh, int elIndex)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
if (elInfo->getElement()->getIndex() == elIndex)
return elInfo->getMacroElement()->getElement();
elInfo = stack.traverseNext(elInfo);
}
return NULL;
}
Element* getParentElement(Mesh *mesh, Element *el) Element* getParentElement(Mesh *mesh, Element *el)
{ {
TraverseStack stack; TraverseStack stack;
...@@ -243,6 +258,22 @@ namespace AMDiS { ...@@ -243,6 +258,22 @@ namespace AMDiS {
} }
void printElementCoords(const FiniteElemSpace *feSpace, Element *el)
{
FUNCNAME("debug:printElementCoords()");
Mesh *mesh = feSpace->getMesh();
for (int i = 0; i <= mesh->getDim(); i++) {
DegreeOfFreedom dof = el->getDOF(i, 0);
WorldVector<double> coords;
mesh->getDofIndexCoords(dof, feSpace, coords);
MSG("%d-th DOF of element %d: %f %f %f\n",
i, el->getIndex(), coords[0], coords[1], coords[2]);
}
}
void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof) void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof)
{ {
FUNCNAME("debug::printInfoByDof()"); FUNCNAME("debug::printInfoByDof()");
...@@ -557,6 +588,48 @@ namespace AMDiS { ...@@ -557,6 +588,48 @@ namespace AMDiS {
} }
void importDofVectorByCoords(DOFVector<double>* vec, std::string filename)
{
DOFVector<WorldVector<double> > coords(vec->getFeSpace(), "dofCoords");
vec->getFeSpace()->getMesh()->getDofIndexCoords(vec->getFeSpace(), coords);
int dim = vec->getFeSpace()->getMesh()->getDim();
std::ifstream file;
file.open(filename.c_str());
int nEntries;
file >> nEntries;
for (int i = 0; i < nEntries; i++) {
WorldVector<double> dofCoords;
for (int j = 0; j < dim; j++)
file >> dofCoords[j];
double value;
file >> value;
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) {
bool found = true;
for (int j = 0; j < dim; j++) {
if (fabs((*it)[j] - dofCoords[j]) > 1e-8) {
found = false;
break;
}
}
if (found) {
(*vec)[it.getDOFIndex()] = value;
break;
}
}
}
file.close();
}
void exportDofVectorByCoords(const DOFVector<double>* vec, void exportDofVectorByCoords(const DOFVector<double>* vec,
std::string filename) std::string filename)
{ {
......
...@@ -86,6 +86,8 @@ namespace AMDiS { ...@@ -86,6 +86,8 @@ namespace AMDiS {
Element* getLevel0ParentElement(Mesh *mesh, Element *el); Element* getLevel0ParentElement(Mesh *mesh, Element *el);
Element* getLevel0ParentElement(Mesh *mesh, int elIndex);
Element* getParentElement(Mesh *mesh, Element *el); Element* getParentElement(Mesh *mesh, Element *el);
Element* getParentElement(Mesh *mesh, int elIndex); Element* getParentElement(Mesh *mesh, int elIndex);
...@@ -93,6 +95,8 @@ namespace AMDiS { ...@@ -93,6 +95,8 @@ namespace AMDiS {
Element* getElement(Mesh *mesh, int elIndex); Element* getElement(Mesh *mesh, int elIndex);
void printElementInfo(Element *el); void printElementInfo(Element *el);
void printElementCoords(const FiniteElemSpace *feSpace, Element *el);
void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof); void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof);
...@@ -141,6 +145,8 @@ namespace AMDiS { ...@@ -141,6 +145,8 @@ namespace AMDiS {
int getLocalNeighbourIndex(Mesh *mesh, int elIndex, int neighIndex); int getLocalNeighbourIndex(Mesh *mesh, int elIndex, int neighIndex);
void importDofVectorByCoords(DOFVector<double>* vec, std::string filename);
void exportDofVectorByCoords(const DOFVector<double>* vec, void exportDofVectorByCoords(const DOFVector<double>* vec,
std::string filename); std::string filename);
......
...@@ -171,7 +171,7 @@ namespace AMDiS { ...@@ -171,7 +171,7 @@ namespace AMDiS {
/// Access to the i-th vector element for const vectors. /// Access to the i-th vector element for const vectors.
inline const T& operator[] (int i) const inline const T& operator[] (int i) const
{ {
TEST_EXIT_DBG(i < size && i >= 0)("invalid index\n"); TEST_EXIT_DBG(i < size && i >= 0)("Invalid index %d!\n", i);
return valArray[i]; return valArray[i];
} }
......
...@@ -214,6 +214,10 @@ namespace AMDiS { ...@@ -214,6 +214,10 @@ namespace AMDiS {
BoundaryObject bound, BoundaryObject bound,
DofContainer& dofs) const DofContainer& dofs) const
{ {
FUNCNAME("Tetrahedron::getVertexDofsAtFace()");
// MSG("go at el %d with face %d\n", this->getIndex(), bound.ithObj);
if (!child[0]) if (!child[0])
return; return;
...@@ -246,7 +250,7 @@ namespace AMDiS { ...@@ -246,7 +250,7 @@ namespace AMDiS {
for (ExcludeList::iterator it = bound.excludedSubstructures.begin(); for (ExcludeList::iterator it = bound.excludedSubstructures.begin();
it != bound.excludedSubstructures.end(); ++it) it != bound.excludedSubstructures.end(); ++it)
if (it->first == EDGE && it->second == 0) if (it->first == EDGE && it->second == 0)
addDof = false; addDof = false;
if (bound.reverseMode) { if (bound.reverseMode) {
child[1]->getVertexDofs(feSpace, nextBound1, dofs); child[1]->getVertexDofs(feSpace, nextBound1, dofs);
...@@ -327,8 +331,8 @@ namespace AMDiS { ...@@ -327,8 +331,8 @@ namespace AMDiS {
case FACE: case FACE:
for (ExcludeList::iterator it = bound.excludedSubstructures.begin(); for (ExcludeList::iterator it = bound.excludedSubstructures.begin();
it != bound.excludedSubstructures.end(); ++it) { it != bound.excludedSubstructures.end(); ++it) {
if (it->first == EDGE) if (it->first == EDGE && it->second != -1)
it->second = edgeOfChild[bound.elType][ithChild][it->second]; it->second = edgeOfChild[bound.elType][ithChild][it->second];
} }
bound.ithObj = sideOfChild[bound.elType][ithChild][bound.ithObj]; bound.ithObj = sideOfChild[bound.elType][ithChild][bound.ithObj];
......
...@@ -68,9 +68,9 @@ namespace AMDiS { ...@@ -68,9 +68,9 @@ namespace AMDiS {
newAssembler = new StandardZOA(op, assembler, quad); newAssembler = new StandardZOA(op, assembler, quad);
} else { } else {
if (pwConst) if (pwConst)
newAssembler = new PrecalcZOA(op, assembler, quad); newAssembler = new PrecalcZOA(op, assembler, quad);
else else
newAssembler = new FastQuadZOA(op, assembler, quad); newAssembler = new FastQuadZOA(op, assembler, quad);
} }
subAssemblers->push_back(newAssembler); subAssemblers->push_back(newAssembler);
......
...@@ -36,7 +36,7 @@ namespace AMDiS { ...@@ -36,7 +36,7 @@ namespace AMDiS {
{ {
if (f) { if (f) {
for (int iq = 0; iq < nPoints; iq++) for (int iq = 0; iq < nPoints; iq++)
C[iq] += (*f)(vecAtQPs[iq]); C[iq] += (*f)(vecAtQPs[iq]);
} else { } else {
for (int iq = 0; iq < nPoints; iq++) for (int iq = 0; iq < nPoints; iq++)
C[iq] += vecAtQPs[iq]; C[iq] += vecAtQPs[iq];
......
...@@ -147,6 +147,7 @@ namespace AMDiS { ...@@ -147,6 +147,7 @@ namespace AMDiS {
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this); ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
MSG("Debug mode tests finished!\n"); MSG("Debug mode tests finished!\n");
debug::writeMesh(feSpace, -1, "macro_mesh"); debug::writeMesh(feSpace, -1, "macro_mesh");
...@@ -595,19 +596,6 @@ namespace AMDiS { ...@@ -595,19 +596,6 @@ namespace AMDiS {
if (elCode.getCode() != recvCodes[i].getCode()) { if (elCode.getCode() != recvCodes[i].getCode()) {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n"); TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
#if 0
MSG("CHECK EL %d %d %d %d WITH El %d %d %d %d in RANK %d\n",
boundIt->rankObj.elIndex,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->neighObj.elIndex,
boundIt->neighObj.subObj,
boundIt->neighObj.ithObj,
boundIt->neighObj.elType,
it->first);
#endif
bool b = startFitElementToMeshCode(recvCodes[i], bool b = startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el, boundIt->rankObj.el,
boundIt->rankObj.subObj, boundIt->rankObj.subObj,
...@@ -699,7 +687,6 @@ namespace AMDiS { ...@@ -699,7 +687,6 @@ namespace AMDiS {
refineManager->setMesh(el->getMesh()); refineManager->setMesh(el->getMesh());
refineManager->setStack(&stack); refineManager->setStack(&stack);
refineManager->refineFunction(elInfo); refineManager->refineFunction(elInfo);
MSG("REFINE EL %d %d %d\n", elInfo->getElement()->getIndex(), el->getChild(0)->getIndex(), el->getChild(1)->getIndex());
meshChanged = true; meshChanged = true;
} }
...@@ -778,7 +765,6 @@ namespace AMDiS { ...@@ -778,7 +765,6 @@ namespace AMDiS {
refineManager->setMesh(el->getMesh()); refineManager->setMesh(el->getMesh());
refineManager->setStack(&stack); refineManager->setStack(&stack);
refineManager->refineFunction(elInfo); refineManager->refineFunction(elInfo);
MSG("REFINE EL %d %d %d\n", elInfo->getElement()->getIndex(), el->getChild(0)->getIndex(), el->getChild(1)->getIndex());
meshChanged = true; meshChanged = true;
} }
...@@ -1221,9 +1207,14 @@ namespace AMDiS { ...@@ -1221,9 +1207,14 @@ namespace AMDiS {
bound.rankObj.subObj = geoIndex; bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject; bound.rankObj.ithObj = rankBoundEl.ithObject;
if (geoIndex == FACE) if (geoIndex == FACE) {
for (int edgeNo = 0; edgeNo < 6; edgeNo++) for (int edgeNo = 0; edgeNo < 3; edgeNo++) {
bound.rankObj.excludedSubstructures.push_back(std::make_pair(EDGE, edgeNo)); int edgeOfFace =
bound.rankObj.el->getEdgeOfFace(bound.rankObj.ithObj, edgeNo);
bound.rankObj.excludedSubstructures.push_back(std::make_pair(EDGE, edgeOfFace));
}
}
if (owner == mpiRank) { if (owner == mpiRank) {
...@@ -1601,7 +1592,7 @@ namespace AMDiS { ...@@ -1601,7 +1592,7 @@ namespace AMDiS {
it->rankObj.el->getNonVertexDofs(feSpace, it->rankObj, dofs); it->rankObj.el->getNonVertexDofs(feSpace, it->rankObj, dofs);
for (unsigned int i = 0; i < dofs.size(); i++) for (unsigned int i = 0; i < dofs.size(); i++)
sendDofs[it.getRank()].push_back(dofs[i]); sendDofs[it.getRank()].push_back(dofs[i]);
} }
...@@ -1638,7 +1629,7 @@ namespace AMDiS { ...@@ -1638,7 +1629,7 @@ namespace AMDiS {
// the dofs in ranks partition that are owned by other rank are set to false. // the dofs in ranks partition that are owned by other rank are set to false.
isRankDof.clear(); isRankDof.clear();
for (int i = 0; i < nRankAllDofs; i++) for (int i = 0; i < nRankAllDofs; i++)
isRankDof[i] = true; isRankDof[i] = true;
// Stores for all rank owned dofs a new global index. // Stores for all rank owned dofs a new global index.
DofIndexMap rankDofsNewGlobalIndex; DofIndexMap rankDofsNewGlobalIndex;
...@@ -1697,8 +1688,14 @@ namespace AMDiS { ...@@ -1697,8 +1688,14 @@ namespace AMDiS {
lastMeshChangeIndex = mesh->getChangeIndex(); lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0) #if (DEBUG != 0)
std::stringstream oss;
oss << "elementIndex-" << mpiRank << ".vtu";
debug::writeElementIndexMesh(mesh, oss.str());
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
ParallelDebug::writeDebugFile(*this, "mpi-dbg", "dat");
#if 0 #if 0
MSG("------------- Debug information -------------\n"); MSG("------------- Debug information -------------\n");
...@@ -1721,16 +1718,6 @@ namespace AMDiS { ...@@ -1721,16 +1718,6 @@ namespace AMDiS {
MSG("%d recv DOF: %d\n", i, *(it->second[i])); MSG("%d recv DOF: %d\n", i, *(it->second[i]));
} }
*/ */
#if 0
std::set<const DegreeOfFreedom*> testDofs;
debug::getAllDofs(feSpace, testDofs);
for (std::set<const DegreeOfFreedom*>::iterator it = testDofs.begin();
it != testDofs.end(); ++it) {
MSG("DOF %d: mapLocalGlobalDofs = %d isRankDof = %d\n", **it,
mapLocalGlobalDofs[**it], isRankDof[**it]);
}
#endif
#endif #endif
#endif #endif
} }
......
...@@ -265,6 +265,59 @@ namespace AMDiS { ...@@ -265,6 +265,59 @@ namespace AMDiS {
} }
void ParallelDebug::testGlobalIndexByCoords(MeshDistributor &pdb)
{
FUNCNAME("ParallelDebug::testGlobalIndexByCoords()");
DOFVector<WorldVector<double> > coords(pdb.feSpace, "tmp");
pdb.mesh->getDofIndexCoords(pdb.feSpace, coords);
typedef std::map<WorldVector<double>, int> CoordsIndexMap;
CoordsIndexMap coordsToIndex;
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it)
coordsToIndex[(*it)] = pdb.mapLocalGlobalDofs[it.getDOFIndex()];
StdMpi<CoordsIndexMap> stdMpi(pdb.mpiComm, true);
for (RankToDofContainer::iterator it = pdb.sendDofs.begin();
it != pdb.sendDofs.end(); ++it)
stdMpi.send(it->first, coordsToIndex);
for (RankToDofContainer::iterator it = pdb.recvDofs.begin();
it != pdb.recvDofs.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication<double>(MPI_DOUBLE);
int foundError = 0;
for (RankToDofContainer::iterator it = pdb.recvDofs.begin();
it != pdb.recvDofs.end(); ++it) {
CoordsIndexMap& otherCoords = stdMpi.getRecvData(it->first);
for (CoordsIndexMap::iterator coordsIt = otherCoords.begin();
coordsIt != otherCoords.end(); ++coordsIt) {
if (coordsToIndex.count(coordsIt->first) == 1 &&
coordsToIndex[coordsIt->first] != coordsIt->second) {
std::stringstream oss;
oss.precision(5);
oss << "DOF at coords ";
for (int i = 0; i < Global::getGeo(WORLD); i++)
oss << coordsIt->first[i] << " ";
oss << " do not fit together on rank "
<< pdb.getMpiRank() << " (global index: "
<< coordsToIndex[coordsIt->first] << " and on rank "
<< it->first << " (global index: " << coordsIt->second << ")";
MSG("[DBG] %s\n", oss.str().c_str());
foundError = 1;
}
}
}
mpi::globalAdd(foundError);
TEST_EXIT(foundError == 0)("Error found on at least on rank!\n");
}
void ParallelDebug::testAllElements(MeshDistributor &pdb) void ParallelDebug::testAllElements(MeshDistributor &pdb)
{ {
FUNCNAME("ParallelDebug::testAllElements()"); FUNCNAME("ParallelDebug::testAllElements()");
...@@ -467,8 +520,8 @@ namespace AMDiS { ...@@ -467,8 +520,8 @@ namespace AMDiS {
} }
void ParallelDebug::writeCoordsFile(MeshDistributor &pdb, void ParallelDebug::writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix) std::string prefix, std::string postfix)
{ {
FUNCNAME("ParallelDebug::writeCoordsFile()"); FUNCNAME("ParallelDebug::writeCoordsFile()");
...@@ -478,16 +531,45 @@ namespace AMDiS { ...@@ -478,16 +531,45 @@ namespace AMDiS {
DOFVector<WorldVector<double> > coords(pdb.feSpace, "tmp"); DOFVector<WorldVector<double> > coords(pdb.feSpace, "tmp");
pdb.mesh->getDofIndexCoords(pdb.feSpace, coords); pdb.mesh->getDofIndexCoords(pdb.feSpace, coords);
typedef std::map<int, std::vector<DegreeOfFreedom> > ElDofMap;
ElDofMap elDofMap;
TraverseStack stack;
const BasisFunction *basisFcts = pdb.feSpace->getBasisFcts();
std::vector<DegreeOfFreedom> localIndices(basisFcts->getNumber());
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
basisFcts->getLocalIndices(elInfo->getElement(),
pdb.feSpace->getAdmin(), localIndices);
elDofMap[elInfo->getElement()->getIndex()] = localIndices;
elInfo = stack.traverseNext(elInfo);
}
// === Write informations about all DOFs. ===
std::ofstream file; std::ofstream file;
file.open(filename.str().c_str()); file.open(filename.str().c_str());
file << coords.getUsedSize() << "\n"; file << coords.getUsedSize() << "\n";
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS); DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) { for (it.reset(); !it.end(); ++it) {
file << pdb.mapLocalGlobalDofs[it.getDOFIndex()]; file << it.getDOFIndex() << " "
<< pdb.mapLocalGlobalDofs[it.getDOFIndex()] << " "
<< pdb.getIsRankDof(it.getDOFIndex());
for (int i = 0; i < pdb.mesh->getDim(); i++) for (int i = 0; i < pdb.mesh->getDim(); i++)
file << " " << (*it)[i]; file << " " << (*it)[i];
file << "\n"; file << "\n";
} }
// === Write to all elements in ranks mesh the included dofs. ===
file << elDofMap.size() << "\n";
file << basisFcts->getNumber() << "\n";
for (ElDofMap::iterator it = elDofMap.begin(); it != elDofMap.end(); ++it) {
file << it->first << "\n";
for (int i = 0; i < basisFcts->getNumber(); i++)
file << it->second[i] << " ";
file << "\n";
}
file.close(); file.close();
} }
......
...@@ -42,7 +42,7 @@ namespace AMDiS { ...@@ -42,7 +42,7 @@ namespace AMDiS {
/** \brief /** \brief
* This function is used for debugging only. It traverses all interior boundaries * This function is used for debugging only. It traverses all interior boundaries
* and compares the dof indices on them with the dof indices of the boundarys * and compares the DOF indices on them with the dof indices of the boundarys
* neighbours. The function fails, when dof indices on an interior boundary do * neighbours. The function fails, when dof indices on an interior boundary do
* not fit together. * not fit together.
* *
...@@ -52,6 +52,16 @@ namespace AMDiS { ...@@ -52,6 +52,16 @@ namespace AMDiS {
*/ */
static void testCommonDofs(MeshDistributor &pdb, bool printCoords = false); static void testCommonDofs(MeshDistributor &pdb, bool printCoords = false);
/** \brief
* This function is used for debugging only. It checks if on all ranks DOFs with
* the same coordinates have the same global index. For this, the function genartes
* on all ranks a list of all DOFs with their coordinates and global indices and
* sends the list to all neighbouring ranks.
*
* \param[in] pdb Parallel problem definition used for debugging.
*/
static void testGlobalIndexByCoords(MeshDistributor &pdb);
/** \brief /** \brief
* Tests if all elements in the macro mesh are memeber of exactly one rank. * Tests if all elements in the macro mesh are memeber of exactly one rank.
* *
...@@ -114,8 +124,8 @@ namespace AMDiS { ...@@ -114,8 +124,8 @@ namespace AMDiS {
*/ */
static void printBoundaryInfo(MeshDistributor &pdb); static void printBoundaryInfo(MeshDistributor &pdb);
static void writeCoordsFile(MeshDistributor &pdb, static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix); std::string prefix, std::string postfix);
}; };
} // namespace AMDiS } // namespace AMDiS
......
...@@ -30,7 +30,7 @@ namespace AMDiS { ...@@ -30,7 +30,7 @@ namespace AMDiS {
clock_t first = clock(); clock_t first = clock();
fillPetscMatrix(systemMatrix, rhs); fillPetscMatrix(systemMatrix, rhs);
solvePetscMatrix(*solution, adaptInfo); solvePetscMatrix(*solution, adaptInfo);
#ifdef _OPENMP #ifdef _OPENMP
INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n", INFO(info, 8)("solution of discrete system needed %.5f seconds system time / %.5f seconds wallclock time\n",
...@@ -94,7 +94,7 @@ namespace AMDiS { ...@@ -94,7 +94,7 @@ namespace AMDiS {
// Calculate the exact position of the column index in the petsc matrix. // Calculate the exact position of the column index in the petsc matrix.