From 7bb406ee78ae9ac9bc10ccf1a036b2bf8e7399e8 Mon Sep 17 00:00:00 2001
From: Thomas Witkowski <thomas.witkowski@gmx.de>
Date: Fri, 3 Dec 2010 15:11:23 +0000
Subject: [PATCH] Wow, mesh repartitioning works...
---
AMDiS/libtool | 22 ++++++------
AMDiS/src/Element.h | 6 ++--
AMDiS/src/Mesh.cc | 5 +++
AMDiS/src/MeshStructure.cc | 13 ++-----
AMDiS/src/RefinementManager.cc | 13 +------
AMDiS/src/RefinementManager2d.cc | 10 +-----
AMDiS/src/io/MacroInfo.cc | 12 +++----
AMDiS/src/parallel/MeshDistributor.cc | 49 ++++++++++++++-------------
AMDiS/src/parallel/MeshDistributor.h | 2 ++
9 files changed, 57 insertions(+), 75 deletions(-)
diff --git a/AMDiS/libtool b/AMDiS/libtool
index 730f91e3..5085abd4 100755
--- a/AMDiS/libtool
+++ b/AMDiS/libtool
@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
-max_cmd_len=1572864
+max_cmd_len=98304
# Object file suffix (normally "o").
objext=o
@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
-LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
+LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
-sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
+sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
-sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
+sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
# Whether dlopen is supported.
dlopen_support=unknown
@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
-CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
-CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
+CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
-compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
+compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
-predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
-postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
+predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
+postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predeps=""
-postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
+postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
-compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
+compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
diff --git a/AMDiS/src/Element.h b/AMDiS/src/Element.h
index 5d67b8f5..21f185ce 100644
--- a/AMDiS/src/Element.h
+++ b/AMDiS/src/Element.h
@@ -282,10 +282,12 @@ namespace AMDiS {
*/
inline void setEstimation(double est, int row)
{
+ FUNCNAME("Element::setEstimation()");
+
if (isLeaf()) {
- TEST_EXIT_DBG(elementData)("leaf element without leaf data\n");
+ TEST_EXIT_DBG(elementData)("Leaf element %d without leaf data!\n", index);
ElementData *ld = elementData->getElementData(ESTIMATABLE);
- TEST_EXIT_DBG(ld)("leaf data not estimatable\n");
+ TEST_EXIT_DBG(ld)("Leaf data %d not estimatable!\n", index);
dynamic_cast<LeafDataEstimatableInterface*>(ld)->
setErrorEstimate(row, est);
diff --git a/AMDiS/src/Mesh.cc b/AMDiS/src/Mesh.cc
index f81a7457..9daeb8ca 100644
--- a/AMDiS/src/Mesh.cc
+++ b/AMDiS/src/Mesh.cc
@@ -331,6 +331,11 @@ namespace AMDiS {
if (!(*macroIt)->getElement()->isLeaf()) {
delete (*macroIt)->getElement()->getChild(0);
delete (*macroIt)->getElement()->getChild(1);
+
+ (*macroIt)->getElement()->child[0] = NULL;
+ (*macroIt)->getElement()->child[1] = NULL;
+
+ (*macroIt)->getElement()->setElementData(elementDataPrototype->clone());
}
}
diff --git a/AMDiS/src/MeshStructure.cc b/AMDiS/src/MeshStructure.cc
index e78226eb..f9440744 100644
--- a/AMDiS/src/MeshStructure.cc
+++ b/AMDiS/src/MeshStructure.cc
@@ -246,7 +246,6 @@ namespace AMDiS {
else
elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
- MSG("FIT 1\n");
while (elInfo) {
Element *element = elInfo->getElement();
@@ -265,6 +264,8 @@ namespace AMDiS {
}
}
+ TEST_EXIT_DBG(element)("Should not happen!\n");
+
if (element->isLeaf() && !isLeafElement()) {
MeshStructure *structure = new MeshStructure();
cont = skipBranch(structure);
@@ -283,10 +284,8 @@ namespace AMDiS {
// refine mesh
bool finished = true;
- MSG("FIT 2\n");
- do {
- MSG("RUN ON EL %d\n", macroElIndex);
+ do {
finished = true;
if (macroElIndex == -1)
elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
@@ -308,23 +307,17 @@ namespace AMDiS {
int oldMeshIndex = mesh->getChangeIndex();
#endif
- MSG("AND REFINE!\n");
-
if (macroElIndex == -1)
manager->refineMesh(mesh);
else
manager->refineMacroElement(mesh, macroElIndex);
- MSG("AND BACK!\n");
-
#if (DEBUG != 0)
TEST_EXIT(oldMeshIndex != mesh->getChangeIndex())
("Mesh has not been changed by refinement procedure!\n");
#endif
}
} while (!finished);
-
- MSG("FIT 3\n");
}
diff --git a/AMDiS/src/RefinementManager.cc b/AMDiS/src/RefinementManager.cc
index 7d12cded..4cf5a6fe 100644
--- a/AMDiS/src/RefinementManager.cc
+++ b/AMDiS/src/RefinementManager.cc
@@ -91,8 +91,6 @@ namespace AMDiS {
while (doMoreRecursiveRefine) {
doMoreRecursiveRefine = false;
- MSG("HERE 1\n");
-
ElInfo *elInfo =
stack->traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
@@ -103,28 +101,19 @@ namespace AMDiS {
doMoreRecursiveRefine || (elInfo->getElement()->getMark() > 1);
elInfo = refineFunction(elInfo);
}
- MSG("WIRKLICH HERE? 1\n");
elInfo = stack->traverseNext(elInfo);
- MSG("WIRKLICH HERE? 2\n");
}
-
- MSG("HERE 2\n");
}
- MSG("HERE 3\n");
-
if (newCoords)
setNewCoords(macroElIndex); // call of sub-class method
- MSG("HERE 4\n");
delete stack;
- MSG("HERE 5\n");
+
nElements -= mesh->getNumberOfLeaves();
- MSG("HERE 6\n");
if (nElements != 0)
aMesh->incChangeIndex();
- MSG("HERE 7\n");
}
}
diff --git a/AMDiS/src/RefinementManager2d.cc b/AMDiS/src/RefinementManager2d.cc
index 9a585034..02bd80a2 100644
--- a/AMDiS/src/RefinementManager2d.cc
+++ b/AMDiS/src/RefinementManager2d.cc
@@ -18,13 +18,11 @@ namespace AMDiS {
ElInfo* RefinementManager2d::refineFunction(ElInfo* elInfo)
{
- FUNCNAME("RefinementManager::refineFunction()");
+ FUNCNAME("RefinementManager2d::refineFunction()");
if (elInfo->getElement()->getMark() <= 0)
return elInfo;
- MSG("RFCT 1\n");
-
bool bound = false;
DegreeOfFreedom *edge[2];
RCNeighbourList* refineList = new RCNeighbourList(2);
@@ -116,8 +114,6 @@ namespace AMDiS {
delete refineList;
- MSG("RFCT 2\n");
-
return elInfo;
}
@@ -137,15 +133,11 @@ namespace AMDiS {
if (el->getFirstChild() && projector && (!el->isNewCoordSet())) {
WorldVector<double> *new_coord = new WorldVector<double>;
- MSG("NU ABER 1\n");
for (int j = 0; j < dow; j++)
(*new_coord)[j] = (elInfo->getCoord(0)[j] + elInfo->getCoord(1)[j]) * 0.5;
- MSG("NU ABER 2\n");
projector->project(*new_coord);
el->setNewCoord(new_coord);
-
- MSG("NU ABER 3\n");
}
}
diff --git a/AMDiS/src/io/MacroInfo.cc b/AMDiS/src/io/MacroInfo.cc
index 2bea515b..2b75f4ca 100644
--- a/AMDiS/src/io/MacroInfo.cc
+++ b/AMDiS/src/io/MacroInfo.cc
@@ -14,9 +14,7 @@ namespace AMDiS {
TEST_EXIT(pmesh)("no mesh\n");
- int dim = pmesh->getDim();
mesh = pmesh;
-
mesh->setNumberOfElements(nElements);
mesh->setNumberOfLeaves(nElements);
mesh->setNumberOfVertices(nVertices);
@@ -63,11 +61,11 @@ namespace AMDiS {
}
- /****************************************************************************/
- /* read_indices() reads dim+1 indices from file into id[0-dim], */
- /* returns true if dim+1 inputs arguments could be read successfully by */
- /* fscanf(), else false */
- /****************************************************************************/
+ /********************************************************************************/
+ /* read_indices() reads dim + 1 indices from file into id[0 - dim], */
+ /* returns true if dim + 1 inputs arguments could be read successfully by */
+ /* fscanf(), else false */
+ /********************************************************************************/
int MacroInfo::read_indices(FILE *file, DimVec<int> &id)
{
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index 9ad76241..1325fce9 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -57,6 +57,7 @@ namespace AMDiS {
writeSerializationFile(false),
repartitioningAllowed(false),
nTimestepsAfterLastRepartitioning(0),
+ repartCounter(0),
lastMeshChangeIndex(0)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
@@ -970,9 +971,15 @@ namespace AMDiS {
if (repartitioning == 0)
return;
- DOFVector<double> tmpa(feSpace, "tmp");
- tmpa.set(mpiRank);
- VtkWriter::writeFile(testVec[0], "before-repartition.vtu");
+ if (repartCounter == 0) {
+ std::stringstream oss;
+ oss << "partitioning-" << repartCounter << ".vtu";
+ DOFVector<double> tmpa(feSpace, "tmp");
+ tmpa.set(mpiRank);
+ VtkWriter::writeFile(&tmpa, oss.str());
+
+ repartCounter++;
+ }
MSG("USED-SIZE A: %d\n", mesh->getDofAdmin(0).getUsedDofs());
@@ -992,7 +999,6 @@ namespace AMDiS {
MacroInfo* minfo = mesh->getMacroFileInfo();
TEST_EXIT_DBG(minfo)("No macro file info!\n");
- MSG("MARK 1\n");
// === Create map that maps macro element indices to pointers to the ===
// === macro elements. ===
@@ -1002,7 +1008,7 @@ namespace AMDiS {
it != minfo->mel.end(); ++it)
elIndexMap[(*it)->getIndex()] = *it;
- MSG("MARK 2\n");
+
// === Create set of all new macro elements this rank will receive from ===
// === other ranks. ===
@@ -1022,7 +1028,6 @@ namespace AMDiS {
it != mesh->endOfMacroElements(); ++it)
allMacroEl.insert(*it);
- MSG("MARK 3\n");
// === Add new macro elements to mesh. ===
for (std::set<MacroElement*>::iterator it = newMacroEl.begin();
@@ -1037,7 +1042,6 @@ namespace AMDiS {
mesh->getMacroElements().push_back(mel);
}
- MSG("MARK 4\n");
// === Send and receive mesh structure codes. ===
std::map<int, MeshCodeVec> sendCodes;
@@ -1058,17 +1062,16 @@ namespace AMDiS {
}
}
}
- MSG("MARK 5\n");
+
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(partitioner->getRecvElements());
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
- MSG("MARK 6\n");
+
StdMpi<std::vector<std::vector<double> > > stdMpi2(mpiComm, true);
stdMpi2.send(sendValues);
stdMpi2.recv(partitioner->getRecvElements());
stdMpi2.startCommunication<double>(MPI_DOUBLE);
- MSG("MARK 7\n");
// === Adapte the new macro elements due to the received mesh structure codes. ===
@@ -1084,20 +1087,15 @@ namespace AMDiS {
for (std::vector<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt) {
- MSG("START EL ADAPT\n");
recvCodes[i].fitMeshToStructure(mesh, refineManager, false, *elIt);
- MSG("END EL ADAPT\n");
- for (unsigned int k = 0; k < testVec.size(); k++) {
- MSG("START VAL ADAPT\n");
+ for (unsigned int k = 0; k < testVec.size(); k++)
recvCodes[i].setMeshStructureValues(mesh, *elIt, testVec[k], recvValues[j++]);
- MSG("END VAL ADAPT\n");
- }
i++;
}
}
- MSG("MARK 8\n");
+
// === Set correct neighbour information on macro elements. ===
for (std::set<MacroElement*>::iterator it = allMacroEl.begin();
@@ -1112,7 +1110,7 @@ namespace AMDiS {
(*it)->setNeighbour(i, elIndexMap[neighIndex]);
}
}
- MSG("MARK 9\n");
+
// === Delete macros from mesh. ===
@@ -1128,22 +1126,25 @@ namespace AMDiS {
deleteMacroElements.insert(elIndexMap[*elIt]);
}
}
- MSG("MARK 10\n");
+
mesh->removeMacroElements(deleteMacroElements, feSpace);
if (mpiRank == 0)
debug::writeElementIndexMesh(mesh, "elementIndexxxx.vtu");
+
// === Remove double DOFs. ===
- MSG("MARK 11\n");
MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(newMacroEl);
- MSG("MARK 12\n");
+
mesh->dofCompress();
- DOFVector<double> tmp(feSpace, "tmp");
- tmp.set(mpiRank);
- VtkWriter::writeFile(testVec[0], "after-repartition.vtu");
+ std::stringstream oss;
+ oss << "partitioning-" << repartCounter << ".vtu";
+ DOFVector<double> tmpa(feSpace, "tmp");
+ tmpa.set(mpiRank);
+ VtkWriter::writeFile(&tmpa, oss.str());
+ repartCounter++;
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
diff --git a/AMDiS/src/parallel/MeshDistributor.h b/AMDiS/src/parallel/MeshDistributor.h
index 7b68386e..3020cde5 100644
--- a/AMDiS/src/parallel/MeshDistributor.h
+++ b/AMDiS/src/parallel/MeshDistributor.h
@@ -567,6 +567,8 @@ namespace AMDiS {
int nTimestepsAfterLastRepartitioning;
+ int repartCounter;
+
/** \brief
* Stores the mesh change index. This is used to recognize changes in the mesh
* structure (e.g. through refinement or coarsening managers).
--
GitLab