Commit 6e466acf authored by Thomas Witkowski's avatar Thomas Witkowski

Work on parallelization, merge to deimos.

parent 979106b8
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=98304
max_cmd_len=1572864
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
......@@ -246,6 +246,8 @@ namespace AMDiS {
else
elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
MSG("FIT 1\n");
while (elInfo) {
Element *element = elInfo->getElement();
......@@ -281,8 +283,10 @@ namespace AMDiS {
// refine mesh
bool finished = true;
MSG("FIT 2\n");
do {
MSG("RUN ON EL %d\n", macroElIndex);
finished = true;
if (macroElIndex == -1)
elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
......@@ -303,18 +307,24 @@ namespace AMDiS {
#if (DEBUG != 0)
int oldMeshIndex = mesh->getChangeIndex();
#endif
MSG("AND REFINE!\n");
if (macroElIndex == -1)
manager->refineMesh(mesh);
else
manager->refineMacroElement(mesh, macroElIndex);
MSG("AND BACK!\n");
#if (DEBUG != 0)
TEST_EXIT(oldMeshIndex != mesh->getChangeIndex())
("Mesh has not been changed by refinement procedure!\n");
#endif
}
} while (!finished);
MSG("FIT 3\n");
}
......@@ -366,19 +376,21 @@ namespace AMDiS {
{
FUNCNAME("MeshStructure::getMeshStructureValues()");
TEST_EXIT_DBG(mesh)("No mesh defined!\n");
TEST_EXIT_DBG(vec)("No DOFVector defined!\n");
values.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
if (elInfo->getLevel() == 0) {
if (elInfo->getLevel() == 0)
for (int i = 0; i < mesh->getGeo(VERTEX); i++)
values.push_back((*vec)[elInfo->getElement()->getDof(i, 0)]);
} else {
if (!elInfo->getElement()->isLeaf())
values.push_back((*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)]);
}
if (!elInfo->getElement()->isLeaf())
values.push_back((*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)]);
elInfo = stack.traverseNext(elInfo);
}
......@@ -392,7 +404,10 @@ namespace AMDiS {
{
FUNCNAME("MeshStructure::setMeshStructureValues()");
TEST_EXIT_DBG(values.size() >= mesh->getGeo(VERTEX))("Should not happen!\n");
TEST_EXIT_DBG(mesh)("No mesh defined!\n");
TEST_EXIT_DBG(vec)("No DOFVector defined!\n");
TEST_EXIT_DBG(static_cast<int>(values.size()) >= mesh->getGeo(VERTEX))
("Should not happen!\n");
unsigned int counter = 0;
......@@ -400,16 +415,15 @@ namespace AMDiS {
ElInfo *elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
if (elInfo->getLevel() == 0) {
if (elInfo->getLevel() == 0)
for (int i = 0; i < mesh->getGeo(VERTEX); i++)
(*vec)[elInfo->getElement()->getDof(i, 0)] = values[counter++];
} else {
if (!elInfo->getElement()->isLeaf()) {
TEST_EXIT_DBG(counter < values.size())("Should not happen!\n");
(*vec)[elInfo->getElement()->getDof(i, 0)] = values[counter++];
(*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)] =
values[counter++];
}
if (!elInfo->getElement()->isLeaf()) {
TEST_EXIT_DBG(counter < values.size())("Should not happen!\n");
(*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)] =
values[counter++];
}
elInfo = stack.traverseNext(elInfo);
......
......@@ -90,6 +90,9 @@ namespace AMDiS {
while (doMoreRecursiveRefine) {
doMoreRecursiveRefine = false;
MSG("HERE 1\n");
ElInfo *elInfo =
stack->traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
......@@ -100,20 +103,28 @@ namespace AMDiS {
doMoreRecursiveRefine || (elInfo->getElement()->getMark() > 1);
elInfo = refineFunction(elInfo);
}
MSG("WIRKLICH HERE? 1\n");
elInfo = stack->traverseNext(elInfo);
MSG("WIRKLICH HERE? 2\n");
}
MSG("HERE 2\n");
}
MSG("HERE 3\n");
if (newCoords)
setNewCoords(); // call of sub-class method
setNewCoords(macroElIndex); // call of sub-class method
MSG("HERE 4\n");
delete stack;
MSG("HERE 5\n");
nElements -= mesh->getNumberOfLeaves();
MSG("HERE 6\n");
if (nElements != 0)
aMesh->incChangeIndex();
MSG("HERE 7\n");
}
}
......@@ -51,7 +51,7 @@ namespace AMDiS {
* Generates new coordinates on curved boundaries. Can be overriden by
* sub classes if used.
*/
virtual void setNewCoords()
virtual void setNewCoords(int macroEl = -1)
{
FUNCNAME("RefinementManager::setNewCoords");
ERROR_EXIT("called for base class!\n");
......
......@@ -138,7 +138,7 @@ namespace AMDiS {
}
}
void RefinementManager1d::setNewCoords()
void RefinementManager1d::setNewCoords(int)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
......
......@@ -43,7 +43,7 @@ namespace AMDiS {
Flag refineMesh(Mesh *aMesh);
/// Implements RefinementManager::setNewCoords
void setNewCoords();
void setNewCoords(int macroEl = -1);
protected:
/// Used by refineMesh while mesh traversal
......
......@@ -23,6 +23,8 @@ namespace AMDiS {
if (elInfo->getElement()->getMark() <= 0)
return elInfo;
MSG("RFCT 1\n");
bool bound = false;
DegreeOfFreedom *edge[2];
RCNeighbourList* refineList = new RCNeighbourList(2);
......@@ -114,12 +116,16 @@ namespace AMDiS {
delete refineList;
MSG("RFCT 2\n");
return elInfo;
}
void RefinementManager2d::newCoordsFct(ElInfo *elInfo)
{
FUNCNAME("RefinementManager2d::newCoordsFct()");
Element *el = elInfo->getElement();
int dow = Global::getGeo(WORLD);
......@@ -131,21 +137,33 @@ namespace AMDiS {
if (el->getFirstChild() && projector && (!el->isNewCoordSet())) {
WorldVector<double> *new_coord = new WorldVector<double>;
MSG("NU ABER 1\n");
for (int j = 0; j < dow; j++)
(*new_coord)[j] = (elInfo->getCoord(0)[j] + elInfo->getCoord(1)[j]) * 0.5;
MSG("NU ABER 2\n");
projector->project(*new_coord);
el->setNewCoord(new_coord);
MSG("NU ABER 3\n");
}
}
void RefinementManager2d::setNewCoords()
void RefinementManager2d::setNewCoords(int macroEl)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS);
ElInfo *elInfo;
if (macroEl == -1)
elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS);
else
elInfo = stack.traverseFirstOneMacro(mesh, macroEl, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS);
while (elInfo) {
newCoordsFct(elInfo);
elInfo = stack.traverseNext(elInfo);
......
......@@ -44,7 +44,7 @@ namespace AMDiS {
void newCoordsFct(ElInfo *elInfo);
/// Implements RefinementManager::setNewCoords
void setNewCoords();
void setNewCoords(int macroEl = -1);
/** \brief
* gets the elements around the refinement edge with vertices node[0] and
......
......@@ -285,17 +285,26 @@ namespace AMDiS {
}
void RefinementManager3d::setNewCoords()
void RefinementManager3d::setNewCoords(int macroEl)
{
if (refList)
delete refList;
refList = new RCNeighbourList(mesh->getMaxEdgeNeigh());
ElInfo *elInfo;
if (macroEl == -1)
elInfo = stack->traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS |
Mesh::FILL_NEIGH);
else
elInfo = stack->traverseFirstOneMacro(mesh, macroEl, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS |
Mesh::FILL_NEIGH);
ElInfo *elInfo = stack->traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS |
Mesh::FILL_NEIGH);
while (elInfo) {
newCoordsFct(elInfo);
elInfo = stack->traverseNext(elInfo);
......
......@@ -44,7 +44,7 @@ namespace AMDiS {
int newCoordsFct(ElInfo *el_info);
/// Implements RefinementManager::setNewCoords
void setNewCoords();
void setNewCoords(int macroEl = -1);
/** \brief
* Gets the elements around the refinement edge with vertices node[0] and
......
......@@ -56,6 +56,7 @@ namespace AMDiS {
deserialized(false),
writeSerializationFile(false),
repartitioningAllowed(false),
nTimestepsAfterLastRepartitioning(0),
lastMeshChangeIndex(0)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
......@@ -547,8 +548,14 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. ===
if (repartitioningAllowed)
repartitionMesh();
nTimestepsAfterLastRepartitioning++;
if (repartitioningAllowed) {
if (nTimestepsAfterLastRepartitioning >= 20) {
repartitionMesh();
nTimestepsAfterLastRepartitioning = 0;
}
}
}
......@@ -965,7 +972,7 @@ namespace AMDiS {
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(tmpa, "before-repartition.vtu");
VtkWriter::writeFile(testVec[0], "before-repartition.vtu");
MSG("USED-SIZE A: %d\n", mesh->getDofAdmin(0).getUsedDofs());
......@@ -985,6 +992,7 @@ namespace AMDiS {
MacroInfo* minfo = mesh->getMacroFileInfo();
TEST_EXIT_DBG(minfo)("No macro file info!\n");
MSG("MARK 1\n");
// === Create map that maps macro element indices to pointers to the ===
// === macro elements. ===
......@@ -994,7 +1002,7 @@ namespace AMDiS {
it != minfo->mel.end(); ++it)
elIndexMap[(*it)->getIndex()] = *it;
MSG("MARK 2\n");
// === Create set of all new macro elements this rank will receive from ===
// === other ranks. ===
......@@ -1014,7 +1022,7 @@ namespace AMDiS {
it != mesh->endOfMacroElements(); ++it)
allMacroEl.insert(*it);
MSG("MARK 3\n");
// === Add new macro elements to mesh. ===
for (std::set<MacroElement*>::iterator it = newMacroEl.begin();
......@@ -1029,7 +1037,7 @@ namespace AMDiS {
mesh->getMacroElements().push_back(mel);
}
MSG("MARK 4\n");
// === Send and receive mesh structure codes. ===
std::map<int, MeshCodeVec> sendCodes;
......@@ -1043,34 +1051,53 @@ namespace AMDiS {
elCode.init(mesh, *elIt);
sendCodes[it->first].push_back(elCode);
std::vector<double> valVec;
elCode.getMeshStructureValues(mesh, *elIt, testVec, valVec);
sendValues[it->first].push_back(valVec);
for (unsigned int i = 0; i < testVec.size(); i++) {
std::vector<double> valVec;
elCode.getMeshStructureValues(mesh, *elIt, testVec[i], valVec);
sendValues[it->first].push_back(valVec);
}
}
}
MSG("MARK 5\n");
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(partitioner->getRecvElements());
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
MSG("MARK 6\n");
StdMpi<std::vector<std::vector<double> > > stdMpi2(mpiComm, true);
stdMpi2.send(sendValues);
stdMpi2.recv(partitioner->getRecvElements());
stdMpi2.startCommunication<double>(MPI_DOUBLE);
MSG("MARK 7\n");
// === Adapte the new macro elements due to the received mesh structure codes. ===
for (std::map<int, std::vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it) {
MeshCodeVec &recvCodes = stdMpi.getRecvData()[it->first];
int i = 0;
std::vector<std::vector<double> > &recvValues = stdMpi2.getRecvData()[it->first];
int i = 0;
int j = 0;
TEST_EXIT_DBG(recvCodes.size() == it->second.size())
("Should not happen!\n");
for (std::vector<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt)
recvCodes[i++].fitMeshToStructure(mesh, refineManager, false, *elIt);
elIt != it->second.end(); ++elIt) {
MSG("START EL ADAPT\n");
recvCodes[i].fitMeshToStructure(mesh, refineManager, false, *elIt);
MSG("END EL ADAPT\n");
for (unsigned int k = 0; k < testVec.size(); k++) {
MSG("START VAL ADAPT\n");
recvCodes[i].setMeshStructureValues(mesh, *elIt, testVec[k], recvValues[j++]);
MSG("END VAL ADAPT\n");
}
i++;
}
}
MSG("MARK 8\n");
// === Set correct neighbour information on macro elements. ===
for (std::set<MacroElement*>::iterator it = allMacroEl.begin();
......@@ -1085,7 +1112,7 @@ namespace AMDiS {
(*it)->setNeighbour(i, elIndexMap[neighIndex]);
}
}
MSG("MARK 9\n");
// === Delete macros from mesh. ===
......@@ -1101,22 +1128,22 @@ namespace AMDiS {
deleteMacroElements.insert(elIndexMap[*elIt]);
}
}
MSG("MARK 10\n");
mesh->removeMacroElements(deleteMacroElements, feSpace);
if (mpiRank == 0)
debug::writeElementIndexMesh(mesh, "elementIndexxxx.vtu");
// === Remove double DOFs. ===
MSG("MARK 11\n");
MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(newMacroEl);
MSG("MARK 12\n");
mesh->dofCompress();
DOFVector<double> tmp(feSpace, "tmp");
tmp.set(mpiRank);
VtkWriter::writeFile(tmp, "after-repartition.vtu");
VtkWriter::writeFile(testVec[0], "after-repartition.vtu");
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
......
......@@ -404,7 +404,7 @@ namespace AMDiS {
}
public:
DOFVector<double>* testVec;
std::vector<DOFVector<double>* > testVec;
protected:
///
......@@ -565,6 +565,8 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed;
int nTimestepsAfterLastRepartitioning;
/** \brief
* Stores the mesh change index. This is used to recognize changes in the mesh
* structure (e.g. through refinement or coarsening managers).
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment