Commit bc5b945f authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Serveral bug fixes for parallelization.

parent 87712cf8
...@@ -82,13 +82,13 @@ AR="ar" ...@@ -82,13 +82,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/usr/lib/openmpi/1.2.7-gcc//bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="gcc" CC="/usr/lib/openmpi/1.2.7-gcc//bin/mpicc"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -171,7 +171,7 @@ dlopen_self=unknown ...@@ -171,7 +171,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6798,13 +6798,13 @@ AR="ar" ...@@ -6798,13 +6798,13 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/usr/lib/openmpi/1.2.7-gcc//bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
# A language-specific compiler. # A language-specific compiler.
CC="g++" CC="/usr/lib/openmpi/1.2.7-gcc//bin/mpiCC"
# Is the compiler the GNU C compiler? # Is the compiler the GNU C compiler?
with_gcc=yes with_gcc=yes
...@@ -6887,7 +6887,7 @@ dlopen_self=unknown ...@@ -6887,7 +6887,7 @@ dlopen_self=unknown
dlopen_self_static=unknown dlopen_self_static=unknown
# Compiler flag to prevent dynamic linking. # Compiler flag to prevent dynamic linking.
link_static_flag="-static" link_static_flag=""
# Compiler flag to turn off builtin functions. # Compiler flag to turn off builtin functions.
no_builtin_flag=" -fno-builtin" no_builtin_flag=" -fno-builtin"
...@@ -6954,11 +6954,11 @@ predeps="" ...@@ -6954,11 +6954,11 @@ predeps=""
# Dependencies to place after the objects being linked to create a # Dependencies to place after the objects being linked to create a
# shared library. # shared library.
postdeps="-lstdc++ -lm -lgcc_s -lc -lgcc_s" postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking # The library search path used internally by the compiler when linking
# a shared library. # a shared library.
compiler_lib_search_path="-L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.." compiler_lib_search_path="-L/usr/lib/openmpi/1.2.7-gcc/lib -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2 -L/usr/lib/gcc/i386-redhat-linux/4.1.2/../../.."
# Method to check whether dependent libraries are shared objects. # Method to check whether dependent libraries are shared objects.
deplibs_check_method="pass_all" deplibs_check_method="pass_all"
...@@ -7103,7 +7103,7 @@ AR="ar" ...@@ -7103,7 +7103,7 @@ AR="ar"
AR_FLAGS="cru" AR_FLAGS="cru"
# A C compiler. # A C compiler.
LTCC="gcc" LTCC="/usr/lib/openmpi/1.2.7-gcc//bin/mpicc"
# LTCC compiler flags. # LTCC compiler flags.
LTCFLAGS="-g -O2" LTCFLAGS="-g -O2"
......
...@@ -432,15 +432,13 @@ namespace AMDiS { ...@@ -432,15 +432,13 @@ namespace AMDiS {
{ {
const BasisFunction *basFct = traverseVector->getFESpace()->getBasisFcts(); const BasisFunction *basFct = traverseVector->getFESpace()->getBasisFcts();
const DOFAdmin* admin = traverseVector->getFESpace()->getAdmin(); const DOFAdmin* admin = traverseVector->getFESpace()->getAdmin();
const DegreeOfFreedom *dof = basFct->getLocalIndices(const_cast<Element *>(elinfo->getElement()), const DegreeOfFreedom *dof =
admin, NULL); basFct->getLocalIndices(const_cast<Element*>(elinfo->getElement()), admin, NULL);
const T *inter_val = const_cast<BasisFunction*>(basFct)->interpol(elinfo, const T *inter_val =
0, const_cast<BasisFunction*>(basFct)->interpol(elinfo, 0, NULL,
NULL, traverseVector->interFct, NULL);
traverseVector->interFct, int nBasFcts = basFct->getNumber();
NULL); for (int i = 0; i < nBasFcts; i++)
int number = basFct->getNumber();
for (int i = 0; i < number; i++)
(*traverseVector)[dof[i]] = inter_val[i]; (*traverseVector)[dof[i]] = inter_val[i];
return 0; return 0;
...@@ -959,7 +957,6 @@ namespace AMDiS { ...@@ -959,7 +957,6 @@ namespace AMDiS {
static T* localVec = NULL; static T* localVec = NULL;
static int localVecSize = 0; static int localVecSize = 0;
const DOFAdmin* admin = feSpace->getAdmin(); const DOFAdmin* admin = feSpace->getAdmin();
T *result; T *result;
if (d) { if (d) {
......
...@@ -258,13 +258,14 @@ namespace AMDiS { ...@@ -258,13 +258,14 @@ namespace AMDiS {
void Element::newDOFFct2(const DOFAdmin* admin) void Element::newDOFFct2(const DOFAdmin* admin)
{ {
int i, j, k, n0, nd, nd0; int i, j, k, n0, nd0;
DegreeOfFreedom *ldof; DegreeOfFreedom *ldof;
int vertices = mesh->getGeo(VERTEX); int vertices = mesh->getGeo(VERTEX);
int edges = mesh->getGeo(EDGE); int edges = mesh->getGeo(EDGE);
int faces = mesh->getGeo(FACE); int faces = mesh->getGeo(FACE);
if (nd = admin->getNumberOfDOFs(VERTEX)) { int nd = admin->getNumberOfDOFs(VERTEX);
if (nd) {
nd0 = admin->getNumberOfPreDOFs(VERTEX); nd0 = admin->getNumberOfPreDOFs(VERTEX);
n0 = admin->getMesh()->getNode(VERTEX); n0 = admin->getMesh()->getNode(VERTEX);
for (i = 0; i < vertices; i++) { for (i = 0; i < vertices; i++) {
...@@ -273,7 +274,8 @@ namespace AMDiS { ...@@ -273,7 +274,8 @@ namespace AMDiS {
} }
if (mesh->getDim() > 1) { if (mesh->getDim() > 1) {
if (nd = admin->getNumberOfDOFs(EDGE)) { nd = admin->getNumberOfDOFs(EDGE);
if (nd) {
nd0 = admin->getNumberOfPreDOFs(EDGE); nd0 = admin->getNumberOfPreDOFs(EDGE);
n0 = admin->getMesh()->getNode(EDGE); n0 = admin->getMesh()->getNode(EDGE);
for (i = 0; i < edges; i++) { for (i = 0; i < edges; i++) {
...@@ -283,7 +285,8 @@ namespace AMDiS { ...@@ -283,7 +285,8 @@ namespace AMDiS {
} }
if (mesh->getDim() == 3) { if (mesh->getDim() == 3) {
if (nd = admin->getNumberOfDOFs(FACE)) { nd = admin->getNumberOfDOFs(FACE);
if (nd) {
nd0 = admin->getNumberOfPreDOFs(FACE); nd0 = admin->getNumberOfPreDOFs(FACE);
n0 = admin->getMesh()->getNode(FACE); n0 = admin->getMesh()->getNode(FACE);
for (i = 0; i < faces; i++) { for (i = 0; i < faces; i++) {
...@@ -292,7 +295,8 @@ namespace AMDiS { ...@@ -292,7 +295,8 @@ namespace AMDiS {
} }
} }
if (nd = admin->getNumberOfDOFs(CENTER)) { nd = admin->getNumberOfDOFs(CENTER);
if (nd) {
nd0 = admin->getNumberOfPreDOFs(CENTER); nd0 = admin->getNumberOfPreDOFs(CENTER);
n0 = admin->getMesh()->getNode(CENTER); n0 = admin->getMesh()->getNode(CENTER);
// only one center // only one center
......
...@@ -62,7 +62,7 @@ namespace AMDiS { ...@@ -62,7 +62,7 @@ namespace AMDiS {
bool nextStrict(); bool nextStrict();
/// Returns the dof index of the current dof. /// Returns the dof index of the current dof.
inline const DegreeOfFreedom getDof() inline DegreeOfFreedom getDof()
{ {
if (inOrder) if (inOrder)
return dofs[node0 + elementPos][n0 + orderPosition[dofPos]]; return dofs[node0 + elementPos][n0 + orderPosition[dofPos]];
......
...@@ -24,6 +24,8 @@ ...@@ -24,6 +24,8 @@
#include "Projection.h" #include "Projection.h"
#include "ElInfoStack.h" #include "ElInfoStack.h"
#include "mpi.h"
namespace AMDiS { namespace AMDiS {
#define TIME_USED(f,s) ((double)((s)-(f))/(double)CLOCKS_PER_SEC) #define TIME_USED(f,s) ((double)((s)-(f))/(double)CLOCKS_PER_SEC)
...@@ -1194,6 +1196,34 @@ namespace AMDiS { ...@@ -1194,6 +1196,34 @@ namespace AMDiS {
macroFileInfo = NULL; macroFileInfo = NULL;
} }
void Mesh::computeMatrixFillin(const FiniteElemSpace *feSpace,
std::vector<int> &nnzInRow, int &overall, int &average)
{
std::map<DegreeOfFreedom, int> dofCounter;
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(this, -1, Mesh::CALL_LEAF_EL);
ElementDofIterator elDofIter(feSpace);
while (elInfo) {
elDofIter.reset(elInfo->getElement());
do {
DegreeOfFreedom dof = elDofIter.getDof();
if (dofCounter.count(dof) == 0) {
dofCounter[dof] = 1;
} else {
dofCounter[dof]++;
}
} while (elDofIter.next());
elInfo = stack.traverseNext(elInfo);
}
overall = 0;
for (std::map<DegreeOfFreedom, int>::iterator it = dofCounter.begin();
it != dofCounter.end(); ++it) {
overall += it->second * 15;
}
}
int Mesh::calcMemoryUsage() int Mesh::calcMemoryUsage()
{ {
int result = sizeof(Mesh); int result = sizeof(Mesh);
......
...@@ -564,6 +564,13 @@ namespace AMDiS { ...@@ -564,6 +564,13 @@ namespace AMDiS {
/// ///
void clearMacroFileInfo(); void clearMacroFileInfo();
/** \brief
* Traverse this mesh to compute the number of non zero elements the assembled
* matrix will have in each row.
*/
void computeMatrixFillin(const FiniteElemSpace *feSpace,
std::vector<int> &nnzInRow, int &overall, int &average);
/// ///
int calcMemoryUsage(); int calcMemoryUsage();
......
...@@ -144,23 +144,8 @@ namespace AMDiS { ...@@ -144,23 +144,8 @@ namespace AMDiS {
DbgTestCommonDofs(true); DbgTestCommonDofs(true);
#endif #endif
// === Create petsc matrix. ===
nRankRows = nRankDOFs * nComponents; nRankRows = nRankDOFs * nComponents;
nOverallRows = nOverallDOFs * nComponents; nOverallRows = nOverallDOFs * nComponents;
VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
VecSetType(petscRhsVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscSolVec);
VecSetSizes(petscSolVec, nRankRows, nOverallRows);
VecSetType(petscSolVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
} }
...@@ -244,21 +229,12 @@ namespace AMDiS { ...@@ -244,21 +229,12 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor), for (icursor_type icursor = begin<nz>(cursor),
icend = end<nz>(cursor); icursor != icend; ++icursor) icend = end<nz>(cursor); icursor != icend; ++icursor)
if (value(*icursor) != 0.0) { if (value(*icursor) != 0.0) {
if (mpiRank == 0 && r == 0)
std::cout << "C = " << col(*icursor)
<< " = " << mapLocalGlobalDOFs[col(*icursor)]
<< " = " << mapLocalGlobalDOFs[col(*icursor)] * dispMult + dispAddCol
<< " -> " << value(*icursor)
<< std::endl;
cols.push_back(mapLocalGlobalDOFs[col(*icursor)] * dispMult + dispAddCol); cols.push_back(mapLocalGlobalDOFs[col(*icursor)] * dispMult + dispAddCol);
values.push_back(value(*icursor)); values.push_back(value(*icursor));
} }
MatSetValues(petscMatrix, 1, &r, cols.size(), &(cols[0]), &(values[0]), ADD_VALUES); MatSetValues(petscMatrix, 1, &r, cols.size(), &(cols[0]), &(values[0]), ADD_VALUES);
} }
std::cout << "==========" << std::endl;
} }
...@@ -281,6 +257,18 @@ namespace AMDiS { ...@@ -281,6 +257,18 @@ namespace AMDiS {
MatSetSizes(petscMatrix, nRankRows, nRankRows, nOverallRows, nOverallRows); MatSetSizes(petscMatrix, nRankRows, nRankRows, nOverallRows, nOverallRows);
MatSetType(petscMatrix, MATAIJ); MatSetType(petscMatrix, MATAIJ);
VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
VecSetType(petscRhsVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscSolVec);
VecSetSizes(petscSolVec, nRankRows, nOverallRows);
VecSetType(petscSolVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
setDofMatrix(mat); setDofMatrix(mat);
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
...@@ -299,6 +287,18 @@ namespace AMDiS { ...@@ -299,6 +287,18 @@ namespace AMDiS {
clock_t first = clock(); clock_t first = clock();
VecCreate(PETSC_COMM_WORLD, &petscRhsVec);
VecSetSizes(petscRhsVec, nRankRows, nOverallRows);
VecSetType(petscRhsVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscSolVec);
VecSetSizes(petscSolVec, nRankRows, nOverallRows);
VecSetType(petscSolVec, VECMPI);
VecCreate(PETSC_COMM_WORLD, &petscTmpVec);
VecSetSizes(petscTmpVec, nRankRows, nOverallRows);
VecSetType(petscTmpVec, VECMPI);
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end; using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
namespace traits= mtl::traits; namespace traits= mtl::traits;
typedef DOFMatrix::base_matrix_type Matrix; typedef DOFMatrix::base_matrix_type Matrix;
...@@ -481,7 +481,7 @@ namespace AMDiS { ...@@ -481,7 +481,7 @@ namespace AMDiS {
} }
i++; i++;
} }
MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows, MatCreateMPIAIJ(PETSC_COMM_WORLD, nRankRows, nRankRows, nOverallRows, nOverallRows,
0, d_nnz, 0, o_nnz, &petscMatrix); 0, d_nnz, 0, o_nnz, &petscMatrix);
...@@ -499,7 +499,7 @@ namespace AMDiS { ...@@ -499,7 +499,7 @@ namespace AMDiS {
for (int i = 0; i < nComponents; i++) for (int i = 0; i < nComponents; i++)
for (int j = 0; j < nComponents; j++) for (int j = 0; j < nComponents; j++)
if ((*mat)[i][j]) if ((*mat)[i][j])
setDofMatrix((*mat)[i][j], nComponents, i, j); setDofMatrix((*mat)[i][j], nComponents, i, j);
MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(petscMatrix, MAT_FINAL_ASSEMBLY);
...@@ -586,6 +586,9 @@ namespace AMDiS { ...@@ -586,6 +586,9 @@ namespace AMDiS {
delete [] sendBuffers[i]; delete [] sendBuffers[i];
MatDestroy(petscMatrix); MatDestroy(petscMatrix);
VecDestroy(petscRhsVec);
VecDestroy(petscSolVec);
VecDestroy(petscTmpVec);
} }
...@@ -611,17 +614,10 @@ namespace AMDiS { ...@@ -611,17 +614,10 @@ namespace AMDiS {
for (int i = 0; i < nComponents; i++) { for (int i = 0; i < nComponents; i++) {
DOFVector<double> *dofvec = vec.getDOFVector(i); DOFVector<double> *dofvec = vec.getDOFVector(i);
for (int j = 0; j < nRankDOFs; j++) { for (int j = 0; j < nRankDOFs; j++)
if (i == 0 && mapLocalToDofIndex[j] == 0) (*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i];
std::cout << "ZUGRIFF AUF: " << j * nComponents + i << std::endl;
(*dofvec)[mapLocalToDofIndex[j]] = vecPointer[j * nComponents + i];
}
} }
if (mpiRank == 0)
std::cout << "RESULT = " << (*(vec.getDOFVector(0)))[0] << std::endl;
VecRestoreArray(petscSolVec, &vecPointer); VecRestoreArray(petscSolVec, &vecPointer);
synchVectors(vec); synchVectors(vec);
...@@ -637,6 +633,9 @@ namespace AMDiS { ...@@ -637,6 +633,9 @@ namespace AMDiS {
MSG(" Residual norm: %e\n", norm); MSG(" Residual norm: %e\n", norm);
MatDestroy(petscMatrix); MatDestroy(petscMatrix);
VecDestroy(petscRhsVec);
VecDestroy(petscSolVec);
VecDestroy(petscTmpVec);
KSPDestroy(solver); KSPDestroy(solver);
} }
...@@ -933,7 +932,6 @@ namespace AMDiS { ...@@ -933,7 +932,6 @@ namespace AMDiS {
int i = 0; int i = 0;
for (DofContainer::iterator dofIt = rankAllDofs.begin(); for (DofContainer::iterator dofIt = rankAllDofs.begin();
dofIt != rankAllDofs.end(); ++dofIt) { dofIt != rankAllDofs.end(); ++dofIt) {
rankDofsNewLocalIndex[*dofIt] = i; rankDofsNewLocalIndex[*dofIt] = i;
// First, we set all dofs in ranks partition to be owend by the rank. Later, // First, we set all dofs in ranks partition to be owend by the rank. Later,
// the dofs in ranks partition that are owned by other rank are set to false. // the dofs in ranks partition that are owned by other rank are set to false.
......
...@@ -32,7 +32,7 @@ namespace AMDiS { ...@@ -32,7 +32,7 @@ namespace AMDiS {
Flag adoptFlag) Flag adoptFlag)
{ {
FUNCNAME("ProblemVec::initialize()"); FUNCNAME("ProblemVec::initialize()");
// === create meshes === // === create meshes ===
if (meshes.size() != 0) { if (meshes.size() != 0) {
WARNING("meshes already created\n"); WARNING("meshes already created\n");
...@@ -646,6 +646,14 @@ namespace AMDiS { ...@@ -646,6 +646,14 @@ namespace AMDiS {
int nnz = 0; int nnz = 0;
for (int i = 0; i < nComponents; i++) { for (int i = 0; i < nComponents; i++) {
#if 0
std::vector<int> nnzInRow;
int overallNnz, averageNnz;
componentSpaces[i]->getMesh()->computeMatrixFillin(componentSpaces[i],
nnzInRow, overallNnz, averageNnz);
#endif
MSG("%d DOFs for %s\n", MSG("%d DOFs for %s\n",
componentSpaces[i]->getAdmin()->getUsedSize(), componentSpaces[i]->getAdmin()->getUsedSize(),
componentSpaces[i]->getName().c_str()); componentSpaces[i]->getName().c_str());
...@@ -1384,7 +1392,7 @@ namespace AMDiS { ...@@ -1384,7 +1392,7 @@ namespace AMDiS {
{ {
const BasisFunction *basisFcts = componentSpaces[0]->getBasisFcts(); const BasisFunction *basisFcts = componentSpaces[0]->getBasisFcts();
WorldVector<double> coords; WorldVector<double> coords;
ElementDofIterator elDofIter(componentSpaces[0]); ElementDofIterator elDofIter(componentSpaces[0], true);
TraverseStack stack; TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(componentMeshes[0], -1, ElInfo *elInfo = stack.traverseFirst(componentMeshes[0], -1,
...@@ -1456,7 +1464,7 @@ namespace AMDiS { ...@@ -1456,7 +1464,7 @@ namespace AMDiS {
for (DofToCoord::iterator it = dofToCoords.begin(); it != dofToCoords.end(); ++it) { for (DofToCoord::iterator it = dofToCoords.begin(); it != dofToCoords.end(); ++it) {
out << it->first << std::endl; out << it->first << std::endl;
for (int j = 0; j < dimOfWorld; j++) for (int j = 0; j < dimOfWorld; j++)
out << (it->second)[j] << std::endl; out << (it->second)[j] << std::endl;
} }
// Write the matrices. // Write the matrices.
...@@ -1491,12 +1499,40 @@ namespace AMDiS { ...@@ -1491,12 +1499,40 @@ namespace AMDiS {
TEST_EXIT(testNnz == nnz)("NNZ does not fit together!\n"); TEST_EXIT(testNnz == nnz)("NNZ does not fit together!\n");
} }
} }
for (int i = 0; i < nComponents; i++) {
out << rhs->getDOFVector(i)->getUsedSize() << std::endl;
DOFIterator<double> it(rhs->getDOFVector(i), USED_DOFS);
int c = 0;
for (it.reset(); !it.end(); ++it) {
out << c << " " << *it << std::endl;
c++;
}
TEST_EXIT(c == rhs->getDOFVector(i)->getUsedSize())
("RHS NNZ does not fit together!\n");
}
for (int i = 0; i < nComponents; i++) {
out << solution->getDOFVector(i)->getUsedSize() << std::endl;
DOFIterator<double> it(solution->getDOFVector(i), USED_DOFS);
int c = 0;
for (it.reset(); !it.end(); ++it) {
out << c << " " << *it << std::endl;
c++;
}
TEST_EXIT(c == rhs->getDOFVector(i)->getUsedSize())
("RHS NNZ does not fit together!\n");
}
out.close(); out.close();
} }
void ProblemVec::readAndCompareDbgMatrix(std::string filename) void ProblemVec::readAndCompareDbgMatrix(std::