Commit a24ca7a5 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Fixed several bugs in parallel code. Added more parallel check and debug functions.

parent f9e40e6c
...@@ -178,6 +178,21 @@ namespace AMDiS { ...@@ -178,6 +178,21 @@ namespace AMDiS {
} }
Element* getLevel0ParentElement(Mesh *mesh, int elIndex)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
if (elInfo->getElement()->getIndex() == elIndex)
return elInfo->getMacroElement()->getElement();
elInfo = stack.traverseNext(elInfo);
}
return NULL;
}
Element* getParentElement(Mesh *mesh, Element *el) Element* getParentElement(Mesh *mesh, Element *el)
{ {
TraverseStack stack; TraverseStack stack;
...@@ -243,6 +258,22 @@ namespace AMDiS { ...@@ -243,6 +258,22 @@ namespace AMDiS {
} }
void printElementCoords(const FiniteElemSpace *feSpace, Element *el)
{
FUNCNAME("debug:printElementCoords()");
Mesh *mesh = feSpace->getMesh();
for (int i = 0; i <= mesh->getDim(); i++) {
DegreeOfFreedom dof = el->getDOF(i, 0);
WorldVector<double> coords;
mesh->getDofIndexCoords(dof, feSpace, coords);
MSG("%d-th DOF of element %d: %f %f %f\n",
i, el->getIndex(), coords[0], coords[1], coords[2]);
}
}
void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof) void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof)
{ {
FUNCNAME("debug::printInfoByDof()"); FUNCNAME("debug::printInfoByDof()");
...@@ -557,6 +588,48 @@ namespace AMDiS { ...@@ -557,6 +588,48 @@ namespace AMDiS {
} }
void importDofVectorByCoords(DOFVector<double>* vec, std::string filename)
{
DOFVector<WorldVector<double> > coords(vec->getFeSpace(), "dofCoords");
vec->getFeSpace()->getMesh()->getDofIndexCoords(vec->getFeSpace(), coords);
int dim = vec->getFeSpace()->getMesh()->getDim();
std::ifstream file;
file.open(filename.c_str());
int nEntries;
file >> nEntries;
for (int i = 0; i < nEntries; i++) {
WorldVector<double> dofCoords;
for (int j = 0; j < dim; j++)
file >> dofCoords[j];
double value;
file >> value;
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) {
bool found = true;
for (int j = 0; j < dim; j++) {
if (fabs((*it)[j] - dofCoords[j]) > 1e-8) {
found = false;
break;
}
}
if (found) {
(*vec)[it.getDOFIndex()] = value;
break;
}
}
}
file.close();
}
void exportDofVectorByCoords(const DOFVector<double>* vec, void exportDofVectorByCoords(const DOFVector<double>* vec,
std::string filename) std::string filename)
{ {
......
...@@ -86,6 +86,8 @@ namespace AMDiS { ...@@ -86,6 +86,8 @@ namespace AMDiS {
Element* getLevel0ParentElement(Mesh *mesh, Element *el); Element* getLevel0ParentElement(Mesh *mesh, Element *el);
Element* getLevel0ParentElement(Mesh *mesh, int elIndex);
Element* getParentElement(Mesh *mesh, Element *el); Element* getParentElement(Mesh *mesh, Element *el);
Element* getParentElement(Mesh *mesh, int elIndex); Element* getParentElement(Mesh *mesh, int elIndex);
...@@ -94,6 +96,8 @@ namespace AMDiS { ...@@ -94,6 +96,8 @@ namespace AMDiS {
void printElementInfo(Element *el); void printElementInfo(Element *el);
void printElementCoords(const FiniteElemSpace *feSpace, Element *el);
void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof); void printInfoByDof(FiniteElemSpace *feSpace, DegreeOfFreedom dof);
void printMatValuesStatistics(Matrix<DOFMatrix*> *mat); void printMatValuesStatistics(Matrix<DOFMatrix*> *mat);
...@@ -141,6 +145,8 @@ namespace AMDiS { ...@@ -141,6 +145,8 @@ namespace AMDiS {
int getLocalNeighbourIndex(Mesh *mesh, int elIndex, int neighIndex); int getLocalNeighbourIndex(Mesh *mesh, int elIndex, int neighIndex);
void importDofVectorByCoords(DOFVector<double>* vec, std::string filename);
void exportDofVectorByCoords(const DOFVector<double>* vec, void exportDofVectorByCoords(const DOFVector<double>* vec,
std::string filename); std::string filename);
......
...@@ -171,7 +171,7 @@ namespace AMDiS { ...@@ -171,7 +171,7 @@ namespace AMDiS {
/// Access to the i-th vector element for const vectors. /// Access to the i-th vector element for const vectors.
inline const T& operator[] (int i) const inline const T& operator[] (int i) const
{ {
TEST_EXIT_DBG(i < size && i >= 0)("invalid index\n"); TEST_EXIT_DBG(i < size && i >= 0)("Invalid index %d!\n", i);
return valArray[i]; return valArray[i];
} }
......
...@@ -214,6 +214,10 @@ namespace AMDiS { ...@@ -214,6 +214,10 @@ namespace AMDiS {
BoundaryObject bound, BoundaryObject bound,
DofContainer& dofs) const DofContainer& dofs) const
{ {
FUNCNAME("Tetrahedron::getVertexDofsAtFace()");
// MSG("go at el %d with face %d\n", this->getIndex(), bound.ithObj);
if (!child[0]) if (!child[0])
return; return;
...@@ -327,7 +331,7 @@ namespace AMDiS { ...@@ -327,7 +331,7 @@ namespace AMDiS {
case FACE: case FACE:
for (ExcludeList::iterator it = bound.excludedSubstructures.begin(); for (ExcludeList::iterator it = bound.excludedSubstructures.begin();
it != bound.excludedSubstructures.end(); ++it) { it != bound.excludedSubstructures.end(); ++it) {
if (it->first == EDGE) if (it->first == EDGE && it->second != -1)
it->second = edgeOfChild[bound.elType][ithChild][it->second]; it->second = edgeOfChild[bound.elType][ithChild][it->second];
} }
......
...@@ -147,6 +147,7 @@ namespace AMDiS { ...@@ -147,6 +147,7 @@ namespace AMDiS {
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testInteriorBoundary(*this); ParallelDebug::testInteriorBoundary(*this);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
MSG("Debug mode tests finished!\n"); MSG("Debug mode tests finished!\n");
debug::writeMesh(feSpace, -1, "macro_mesh"); debug::writeMesh(feSpace, -1, "macro_mesh");
...@@ -595,19 +596,6 @@ namespace AMDiS { ...@@ -595,19 +596,6 @@ namespace AMDiS {
if (elCode.getCode() != recvCodes[i].getCode()) { if (elCode.getCode() != recvCodes[i].getCode()) {
TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n"); TEST_EXIT_DBG(refineManager)("Refinement manager is not set correctly!\n");
#if 0
MSG("CHECK EL %d %d %d %d WITH El %d %d %d %d in RANK %d\n",
boundIt->rankObj.elIndex,
boundIt->rankObj.subObj,
boundIt->rankObj.ithObj,
boundIt->rankObj.elType,
boundIt->neighObj.elIndex,
boundIt->neighObj.subObj,
boundIt->neighObj.ithObj,
boundIt->neighObj.elType,
it->first);
#endif
bool b = startFitElementToMeshCode(recvCodes[i], bool b = startFitElementToMeshCode(recvCodes[i],
boundIt->rankObj.el, boundIt->rankObj.el,
boundIt->rankObj.subObj, boundIt->rankObj.subObj,
...@@ -699,7 +687,6 @@ namespace AMDiS { ...@@ -699,7 +687,6 @@ namespace AMDiS {
refineManager->setMesh(el->getMesh()); refineManager->setMesh(el->getMesh());
refineManager->setStack(&stack); refineManager->setStack(&stack);
refineManager->refineFunction(elInfo); refineManager->refineFunction(elInfo);
MSG("REFINE EL %d %d %d\n", elInfo->getElement()->getIndex(), el->getChild(0)->getIndex(), el->getChild(1)->getIndex());
meshChanged = true; meshChanged = true;
} }
...@@ -778,7 +765,6 @@ namespace AMDiS { ...@@ -778,7 +765,6 @@ namespace AMDiS {
refineManager->setMesh(el->getMesh()); refineManager->setMesh(el->getMesh());
refineManager->setStack(&stack); refineManager->setStack(&stack);
refineManager->refineFunction(elInfo); refineManager->refineFunction(elInfo);
MSG("REFINE EL %d %d %d\n", elInfo->getElement()->getIndex(), el->getChild(0)->getIndex(), el->getChild(1)->getIndex());
meshChanged = true; meshChanged = true;
} }
...@@ -1221,9 +1207,14 @@ namespace AMDiS { ...@@ -1221,9 +1207,14 @@ namespace AMDiS {
bound.rankObj.subObj = geoIndex; bound.rankObj.subObj = geoIndex;
bound.rankObj.ithObj = rankBoundEl.ithObject; bound.rankObj.ithObj = rankBoundEl.ithObject;
if (geoIndex == FACE) if (geoIndex == FACE) {
for (int edgeNo = 0; edgeNo < 6; edgeNo++) for (int edgeNo = 0; edgeNo < 3; edgeNo++) {
bound.rankObj.excludedSubstructures.push_back(std::make_pair(EDGE, edgeNo)); int edgeOfFace =
bound.rankObj.el->getEdgeOfFace(bound.rankObj.ithObj, edgeNo);
bound.rankObj.excludedSubstructures.push_back(std::make_pair(EDGE, edgeOfFace));
}
}
if (owner == mpiRank) { if (owner == mpiRank) {
...@@ -1697,8 +1688,14 @@ namespace AMDiS { ...@@ -1697,8 +1688,14 @@ namespace AMDiS {
lastMeshChangeIndex = mesh->getChangeIndex(); lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0) #if (DEBUG != 0)
std::stringstream oss;
oss << "elementIndex-" << mpiRank << ".vtu";
debug::writeElementIndexMesh(mesh, oss.str());
debug::testSortedDofs(mesh, elMap); debug::testSortedDofs(mesh, elMap);
ParallelDebug::testCommonDofs(*this, true); ParallelDebug::testCommonDofs(*this, true);
ParallelDebug::testGlobalIndexByCoords(*this);
ParallelDebug::writeDebugFile(*this, "mpi-dbg", "dat");
#if 0 #if 0
MSG("------------- Debug information -------------\n"); MSG("------------- Debug information -------------\n");
...@@ -1721,16 +1718,6 @@ namespace AMDiS { ...@@ -1721,16 +1718,6 @@ namespace AMDiS {
MSG("%d recv DOF: %d\n", i, *(it->second[i])); MSG("%d recv DOF: %d\n", i, *(it->second[i]));
} }
*/ */
#if 0
std::set<const DegreeOfFreedom*> testDofs;
debug::getAllDofs(feSpace, testDofs);
for (std::set<const DegreeOfFreedom*>::iterator it = testDofs.begin();
it != testDofs.end(); ++it) {
MSG("DOF %d: mapLocalGlobalDofs = %d isRankDof = %d\n", **it,
mapLocalGlobalDofs[**it], isRankDof[**it]);
}
#endif
#endif #endif
#endif #endif
} }
......
...@@ -265,6 +265,59 @@ namespace AMDiS { ...@@ -265,6 +265,59 @@ namespace AMDiS {
} }
void ParallelDebug::testGlobalIndexByCoords(MeshDistributor &pdb)
{
FUNCNAME("ParallelDebug::testGlobalIndexByCoords()");
DOFVector<WorldVector<double> > coords(pdb.feSpace, "tmp");
pdb.mesh->getDofIndexCoords(pdb.feSpace, coords);
typedef std::map<WorldVector<double>, int> CoordsIndexMap;
CoordsIndexMap coordsToIndex;
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it)
coordsToIndex[(*it)] = pdb.mapLocalGlobalDofs[it.getDOFIndex()];
StdMpi<CoordsIndexMap> stdMpi(pdb.mpiComm, true);
for (RankToDofContainer::iterator it = pdb.sendDofs.begin();
it != pdb.sendDofs.end(); ++it)
stdMpi.send(it->first, coordsToIndex);
for (RankToDofContainer::iterator it = pdb.recvDofs.begin();
it != pdb.recvDofs.end(); ++it)
stdMpi.recv(it->first);
stdMpi.startCommunication<double>(MPI_DOUBLE);
int foundError = 0;
for (RankToDofContainer::iterator it = pdb.recvDofs.begin();
it != pdb.recvDofs.end(); ++it) {
CoordsIndexMap& otherCoords = stdMpi.getRecvData(it->first);
for (CoordsIndexMap::iterator coordsIt = otherCoords.begin();
coordsIt != otherCoords.end(); ++coordsIt) {
if (coordsToIndex.count(coordsIt->first) == 1 &&
coordsToIndex[coordsIt->first] != coordsIt->second) {
std::stringstream oss;
oss.precision(5);
oss << "DOF at coords ";
for (int i = 0; i < Global::getGeo(WORLD); i++)
oss << coordsIt->first[i] << " ";
oss << " do not fit together on rank "
<< pdb.getMpiRank() << " (global index: "
<< coordsToIndex[coordsIt->first] << " and on rank "
<< it->first << " (global index: " << coordsIt->second << ")";
MSG("[DBG] %s\n", oss.str().c_str());
foundError = 1;
}
}
}
mpi::globalAdd(foundError);
TEST_EXIT(foundError == 0)("Error found on at least on rank!\n");
}
void ParallelDebug::testAllElements(MeshDistributor &pdb) void ParallelDebug::testAllElements(MeshDistributor &pdb)
{ {
FUNCNAME("ParallelDebug::testAllElements()"); FUNCNAME("ParallelDebug::testAllElements()");
...@@ -467,7 +520,7 @@ namespace AMDiS { ...@@ -467,7 +520,7 @@ namespace AMDiS {
} }
void ParallelDebug::writeCoordsFile(MeshDistributor &pdb, void ParallelDebug::writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix) std::string prefix, std::string postfix)
{ {
FUNCNAME("ParallelDebug::writeCoordsFile()"); FUNCNAME("ParallelDebug::writeCoordsFile()");
...@@ -478,16 +531,45 @@ namespace AMDiS { ...@@ -478,16 +531,45 @@ namespace AMDiS {
DOFVector<WorldVector<double> > coords(pdb.feSpace, "tmp"); DOFVector<WorldVector<double> > coords(pdb.feSpace, "tmp");
pdb.mesh->getDofIndexCoords(pdb.feSpace, coords); pdb.mesh->getDofIndexCoords(pdb.feSpace, coords);
typedef std::map<int, std::vector<DegreeOfFreedom> > ElDofMap;
ElDofMap elDofMap;
TraverseStack stack;
const BasisFunction *basisFcts = pdb.feSpace->getBasisFcts();
std::vector<DegreeOfFreedom> localIndices(basisFcts->getNumber());
ElInfo *elInfo = stack.traverseFirst(pdb.mesh, -1, Mesh::CALL_LEAF_EL);
while (elInfo) {
basisFcts->getLocalIndices(elInfo->getElement(),
pdb.feSpace->getAdmin(), localIndices);
elDofMap[elInfo->getElement()->getIndex()] = localIndices;
elInfo = stack.traverseNext(elInfo);
}
// === Write informations about all DOFs. ===
std::ofstream file; std::ofstream file;
file.open(filename.str().c_str()); file.open(filename.str().c_str());
file << coords.getUsedSize() << "\n"; file << coords.getUsedSize() << "\n";
DOFIterator<WorldVector<double> > it(&coords, USED_DOFS); DOFIterator<WorldVector<double> > it(&coords, USED_DOFS);
for (it.reset(); !it.end(); ++it) { for (it.reset(); !it.end(); ++it) {
file << pdb.mapLocalGlobalDofs[it.getDOFIndex()]; file << it.getDOFIndex() << " "
<< pdb.mapLocalGlobalDofs[it.getDOFIndex()] << " "
<< pdb.getIsRankDof(it.getDOFIndex());
for (int i = 0; i < pdb.mesh->getDim(); i++) for (int i = 0; i < pdb.mesh->getDim(); i++)
file << " " << (*it)[i]; file << " " << (*it)[i];
file << "\n"; file << "\n";
} }
// === Write to all elements in ranks mesh the included dofs. ===
file << elDofMap.size() << "\n";
file << basisFcts->getNumber() << "\n";
for (ElDofMap::iterator it = elDofMap.begin(); it != elDofMap.end(); ++it) {
file << it->first << "\n";
for (int i = 0; i < basisFcts->getNumber(); i++)
file << it->second[i] << " ";
file << "\n";
}
file.close(); file.close();
} }
......
...@@ -42,7 +42,7 @@ namespace AMDiS { ...@@ -42,7 +42,7 @@ namespace AMDiS {
/** \brief /** \brief
* This function is used for debugging only. It traverses all interior boundaries * This function is used for debugging only. It traverses all interior boundaries
* and compares the dof indices on them with the dof indices of the boundarys * and compares the DOF indices on them with the dof indices of the boundarys
* neighbours. The function fails, when dof indices on an interior boundary do * neighbours. The function fails, when dof indices on an interior boundary do
* not fit together. * not fit together.
* *
...@@ -52,6 +52,16 @@ namespace AMDiS { ...@@ -52,6 +52,16 @@ namespace AMDiS {
*/ */
static void testCommonDofs(MeshDistributor &pdb, bool printCoords = false); static void testCommonDofs(MeshDistributor &pdb, bool printCoords = false);
/** \brief
* This function is used for debugging only. It checks if on all ranks DOFs with
* the same coordinates have the same global index. For this, the function genartes
* on all ranks a list of all DOFs with their coordinates and global indices and
* sends the list to all neighbouring ranks.
*
* \param[in] pdb Parallel problem definition used for debugging.
*/
static void testGlobalIndexByCoords(MeshDistributor &pdb);
/** \brief /** \brief
* Tests if all elements in the macro mesh are memeber of exactly one rank. * Tests if all elements in the macro mesh are memeber of exactly one rank.
* *
...@@ -114,7 +124,7 @@ namespace AMDiS { ...@@ -114,7 +124,7 @@ namespace AMDiS {
*/ */
static void printBoundaryInfo(MeshDistributor &pdb); static void printBoundaryInfo(MeshDistributor &pdb);
static void writeCoordsFile(MeshDistributor &pdb, static void writeDebugFile(MeshDistributor &pdb,
std::string prefix, std::string postfix); std::string prefix, std::string postfix);
}; };
......
...@@ -94,7 +94,7 @@ namespace AMDiS { ...@@ -94,7 +94,7 @@ namespace AMDiS {
// Calculate the exact position of the column index in the petsc matrix. // Calculate the exact position of the column index in the petsc matrix.
int colIndex = globalColDof * dispMult + dispAddCol; int colIndex = globalColDof * dispMult + dispAddCol;
// Set only non null values. // Set only non zero values.
if (value(*icursor) != 0.0 || rowIndex == colIndex) { if (value(*icursor) != 0.0 || rowIndex == colIndex) {
// If the current row is not periodic, but the current dof index is periodic, // If the current row is not periodic, but the current dof index is periodic,
...@@ -409,7 +409,6 @@ namespace AMDiS { ...@@ -409,7 +409,6 @@ namespace AMDiS {
// === Create PETSc matrix with the computed nnz data structure. === // === Create PETSc matrix with the computed nnz data structure. ===
MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows, MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
// 0, PETSC_NULL, 0, PETSC_NULL, &petscMatrix);
0, d_nnz, 0, o_nnz, &petscMatrix); 0, d_nnz, 0, o_nnz, &petscMatrix);
#if (DEBUG != 0) #if (DEBUG != 0)
...@@ -481,10 +480,6 @@ namespace AMDiS { ...@@ -481,10 +480,6 @@ namespace AMDiS {
// Do not delete the solution vector, use it for the initial guess. // Do not delete the solution vector, use it for the initial guess.
// KSPSetInitialGuessNonzero(solver, PETSC_TRUE); // KSPSetInitialGuessNonzero(solver, PETSC_TRUE);
#if (DEBUG != 0)
// ParallelDomainDbg::writeCoordsFile(*meshDistributor, "mpi-coords", "dat");
#endif
// === Run Petsc. === // === Run Petsc. ===
......
...@@ -12,6 +12,11 @@ namespace AMDiS { ...@@ -12,6 +12,11 @@ namespace AMDiS {
return data.size() * 2; return data.size() * 2;
} }
int intSizeOf(std::map<WorldVector<double>, int> &data)
{
return data.size() * (Global::getGeo(WORLD) + 1);
}
int intSizeOf(std::vector<MeshStructure> &data) int intSizeOf(std::vector<MeshStructure> &data)
{ {
int s = 0; int s = 0;
...@@ -91,6 +96,38 @@ namespace AMDiS { ...@@ -91,6 +96,38 @@ namespace AMDiS {
data.push_back(std::make_pair(buf[i * 2], buf[i * 2 + 1])); data.push_back(std::make_pair(buf[i * 2], buf[i * 2 + 1]));
} }
void makeBuf(std::map<WorldVector<double>, int> &data, double* buf)
{
int i = 0;
for (std::map<WorldVector<double>, int>::iterator it = data.begin();
it != data.end(); ++it) {
for (int j = 0; j < Global::getGeo(WORLD); j++)
buf[i++] = it->first[j];
buf[i++] = static_cast<double>(it->second);
}
}
void makeFromBuf(std::map<WorldVector<double>, int> &data, double* buf, int bufSize)
{
if (bufSize == 0)
return;
int oneEntrySize = Global::getGeo(WORLD) + 1;
int nEntry = bufSize / oneEntrySize;
TEST_EXIT(bufSize % oneEntrySize == 0)("This should not happen!\n");
data.clear();
int i = 0;
WorldVector<double> coords;
for (int j = 0; j < nEntry; j++) {
for (int k = 0; k < Global::getGeo(WORLD); k++)
coords[k] = buf[i++];