Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer,
es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konten der externen Nutzer:innen sind über den Reiter "Standard" erreichbar.
Die Administratoren


Dear Gitlab user,
it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab.
The administrators

Commit 6e466acf authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on parallelization, merge to deimos.

parent 979106b8
......@@ -85,7 +85,7 @@ NM="/usr/bin/nm -B"
LN_S="ln -s"
# What is the maximum length of a command?
max_cmd_len=98304
max_cmd_len=1572864
# Object file suffix (normally "o").
objext=o
......@@ -128,7 +128,7 @@ old_postinstall_cmds="chmod 644 \$oldlib~\$RANLIB \$oldlib"
old_postuninstall_cmds=""
# A C compiler.
LTCC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
LTCC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# LTCC compiler flags.
LTCFLAGS="-g -O2"
......@@ -233,10 +233,10 @@ finish_eval=""
hardcode_into_libs=yes
# Compile-time system search path for libraries.
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64 /lib64 /fastfs/wir/local/lib /usr/x86_64-suse-linux/lib"
sys_lib_search_path_spec="/usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64 /lib64 /usr/x86_64-suse-linux/lib"
# Run-time system search path for libraries.
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/x86_64-suse-linux/lib /usr/local/lib64 /usr/local/lib /opt/kde3/lib64 /opt/kde3/lib /opt/gnome/lib64 /opt/gnome/lib /lib64 /lib /usr/lib64 /usr/lib /opt/cluster/intel/cce/9.1.042/lib /opt/cluster/intel/fce/9.1.036/lib /opt/cluster/Pathscale3.0/lib/2.9.99 /opt/cluster/Pathscale3.0/lib/2.9.99/32 /work/licsoft/compilers/pgi/linux86-64/6.2/lib /work/licsoft/compilers/pgi/linux86-64/6.2/libso "
sys_lib_dlsearch_path_spec="/lib /usr/lib /usr/X11R6/lib64/Xaw3d /usr/X11R6/lib64 /usr/lib64/Xaw3d /usr/X11R6/lib/Xaw3d /usr/X11R6/lib /usr/lib/Xaw3d /usr/x86_64-suse-linux/lib /usr/local/lib /opt/kde3/lib /lib64 /lib /usr/lib64 /usr/lib /usr/local/lib64 /opt/kde3/lib64 /usr/lib64/graphviz /usr/lib64/graphviz/sharp /usr/lib64/graphviz/java /usr/lib64/graphviz/perl /usr/lib64/graphviz/php /usr/lib64/graphviz/ocaml /usr/lib64/graphviz/python /usr/lib64/graphviz/lua /usr/lib64/graphviz/tcl /usr/lib64/graphviz/guile /usr/lib64/graphviz/ruby /usr/lib64/octave-3.2.4 "
# Whether dlopen is supported.
dlopen_support=unknown
......@@ -259,7 +259,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicc"
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicc"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -8914,7 +8914,7 @@ LD="/usr/x86_64-suse-linux/bin/ld -m elf_x86_64"
old_archive_cmds="\$AR \$AR_FLAGS \$oldlib\$oldobjs~\$RANLIB \$oldlib"
# A language specific compiler.
CC="/licsoft/libraries/openmpi/1.2.6/64bit/bin/mpicxx"
CC="/usr/lib64/mpi/gcc/openmpi//bin/mpicxx"
# Is the compiler the GNU compiler?
with_gcc=yes
......@@ -9039,17 +9039,17 @@ file_list_spec=""
hardcode_action=immediate
# The directories searched by this compiler when creating a shared library.
compiler_lib_search_dirs="/usr/lib64 /licsoft/libraries/openmpi/1.2.6/64bit/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2 /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /fastfs/wir/local/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
compiler_lib_search_dirs="/usr/lib64/mpi/gcc/openmpi/lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 /lib/../lib64 /usr/lib/../lib64 /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib /usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# Dependencies to place before and after the objects being linked to
# create a shared library.
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.1.2/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64/crtn.o"
predep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crti.o /usr/lib64/gcc/x86_64-suse-linux/4.5/crtbeginS.o"
postdep_objects="/usr/lib64/gcc/x86_64-suse-linux/4.5/crtendS.o /usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64/crtn.o"
predeps=""
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -libverbs -lrt -lnuma -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
postdeps="-lmpi_cxx -lmpi -lopen-rte -lopen-pal -ldl -lnsl -lutil -ldl -lstdc++ -lm -lgcc_s -lpthread -lc -lgcc_s"
# The library search path used internally by the compiler when linking
# a shared library.
compiler_lib_search_path="-L/usr/lib64 -L/licsoft/libraries/openmpi/1.2.6/64bit/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2 -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/fastfs/wir/local/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.1.2/../../.."
compiler_lib_search_path="-L/usr/lib64/mpi/gcc/openmpi/lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../../../x86_64-suse-linux/lib -L/usr/lib64/gcc/x86_64-suse-linux/4.5/../../.."
# ### END LIBTOOL TAG CONFIG: CXX
......@@ -246,6 +246,8 @@ namespace AMDiS {
else
elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1, Mesh::CALL_EVERY_EL_PREORDER);
MSG("FIT 1\n");
while (elInfo) {
Element *element = elInfo->getElement();
......@@ -281,8 +283,10 @@ namespace AMDiS {
// refine mesh
bool finished = true;
MSG("FIT 2\n");
do {
MSG("RUN ON EL %d\n", macroElIndex);
finished = true;
if (macroElIndex == -1)
elInfo = stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL);
......@@ -303,18 +307,24 @@ namespace AMDiS {
#if (DEBUG != 0)
int oldMeshIndex = mesh->getChangeIndex();
#endif
MSG("AND REFINE!\n");
if (macroElIndex == -1)
manager->refineMesh(mesh);
else
manager->refineMacroElement(mesh, macroElIndex);
MSG("AND BACK!\n");
#if (DEBUG != 0)
TEST_EXIT(oldMeshIndex != mesh->getChangeIndex())
("Mesh has not been changed by refinement procedure!\n");
#endif
}
} while (!finished);
MSG("FIT 3\n");
}
......@@ -366,19 +376,21 @@ namespace AMDiS {
{
FUNCNAME("MeshStructure::getMeshStructureValues()");
TEST_EXIT_DBG(mesh)("No mesh defined!\n");
TEST_EXIT_DBG(vec)("No DOFVector defined!\n");
values.clear();
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
if (elInfo->getLevel() == 0) {
if (elInfo->getLevel() == 0)
for (int i = 0; i < mesh->getGeo(VERTEX); i++)
values.push_back((*vec)[elInfo->getElement()->getDof(i, 0)]);
} else {
if (!elInfo->getElement()->isLeaf())
values.push_back((*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)]);
}
if (!elInfo->getElement()->isLeaf())
values.push_back((*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)]);
elInfo = stack.traverseNext(elInfo);
}
......@@ -392,7 +404,10 @@ namespace AMDiS {
{
FUNCNAME("MeshStructure::setMeshStructureValues()");
TEST_EXIT_DBG(values.size() >= mesh->getGeo(VERTEX))("Should not happen!\n");
TEST_EXIT_DBG(mesh)("No mesh defined!\n");
TEST_EXIT_DBG(vec)("No DOFVector defined!\n");
TEST_EXIT_DBG(static_cast<int>(values.size()) >= mesh->getGeo(VERTEX))
("Should not happen!\n");
unsigned int counter = 0;
......@@ -400,16 +415,15 @@ namespace AMDiS {
ElInfo *elInfo = stack.traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_EVERY_EL_PREORDER);
while (elInfo) {
if (elInfo->getLevel() == 0) {
if (elInfo->getLevel() == 0)
for (int i = 0; i < mesh->getGeo(VERTEX); i++)
(*vec)[elInfo->getElement()->getDof(i, 0)] = values[counter++];
} else {
if (!elInfo->getElement()->isLeaf()) {
TEST_EXIT_DBG(counter < values.size())("Should not happen!\n");
(*vec)[elInfo->getElement()->getDof(i, 0)] = values[counter++];
(*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)] =
values[counter++];
}
if (!elInfo->getElement()->isLeaf()) {
TEST_EXIT_DBG(counter < values.size())("Should not happen!\n");
(*vec)[elInfo->getElement()->getChild(0)->getDof(mesh->getDim(), 0)] =
values[counter++];
}
elInfo = stack.traverseNext(elInfo);
......
......@@ -90,6 +90,9 @@ namespace AMDiS {
while (doMoreRecursiveRefine) {
doMoreRecursiveRefine = false;
MSG("HERE 1\n");
ElInfo *elInfo =
stack->traverseFirstOneMacro(mesh, macroElIndex, -1,
Mesh::CALL_LEAF_EL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
......@@ -100,20 +103,28 @@ namespace AMDiS {
doMoreRecursiveRefine || (elInfo->getElement()->getMark() > 1);
elInfo = refineFunction(elInfo);
}
MSG("WIRKLICH HERE? 1\n");
elInfo = stack->traverseNext(elInfo);
MSG("WIRKLICH HERE? 2\n");
}
MSG("HERE 2\n");
}
MSG("HERE 3\n");
if (newCoords)
setNewCoords(); // call of sub-class method
setNewCoords(macroElIndex); // call of sub-class method
MSG("HERE 4\n");
delete stack;
MSG("HERE 5\n");
nElements -= mesh->getNumberOfLeaves();
MSG("HERE 6\n");
if (nElements != 0)
aMesh->incChangeIndex();
MSG("HERE 7\n");
}
}
......@@ -51,7 +51,7 @@ namespace AMDiS {
* Generates new coordinates on curved boundaries. Can be overriden by
* sub classes if used.
*/
virtual void setNewCoords()
virtual void setNewCoords(int macroEl = -1)
{
FUNCNAME("RefinementManager::setNewCoords");
ERROR_EXIT("called for base class!\n");
......
......@@ -138,7 +138,7 @@ namespace AMDiS {
}
}
void RefinementManager1d::setNewCoords()
void RefinementManager1d::setNewCoords(int)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
......
......@@ -43,7 +43,7 @@ namespace AMDiS {
Flag refineMesh(Mesh *aMesh);
/// Implements RefinementManager::setNewCoords
void setNewCoords();
void setNewCoords(int macroEl = -1);
protected:
/// Used by refineMesh while mesh traversal
......
......@@ -23,6 +23,8 @@ namespace AMDiS {
if (elInfo->getElement()->getMark() <= 0)
return elInfo;
MSG("RFCT 1\n");
bool bound = false;
DegreeOfFreedom *edge[2];
RCNeighbourList* refineList = new RCNeighbourList(2);
......@@ -114,12 +116,16 @@ namespace AMDiS {
delete refineList;
MSG("RFCT 2\n");
return elInfo;
}
void RefinementManager2d::newCoordsFct(ElInfo *elInfo)
{
FUNCNAME("RefinementManager2d::newCoordsFct()");
Element *el = elInfo->getElement();
int dow = Global::getGeo(WORLD);
......@@ -131,21 +137,33 @@ namespace AMDiS {
if (el->getFirstChild() && projector && (!el->isNewCoordSet())) {
WorldVector<double> *new_coord = new WorldVector<double>;
MSG("NU ABER 1\n");
for (int j = 0; j < dow; j++)
(*new_coord)[j] = (elInfo->getCoord(0)[j] + elInfo->getCoord(1)[j]) * 0.5;
MSG("NU ABER 2\n");
projector->project(*new_coord);
el->setNewCoord(new_coord);
MSG("NU ABER 3\n");
}
}
void RefinementManager2d::setNewCoords()
void RefinementManager2d::setNewCoords(int macroEl)
{
TraverseStack stack;
ElInfo *elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS);
ElInfo *elInfo;
if (macroEl == -1)
elInfo = stack.traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS);
else
elInfo = stack.traverseFirstOneMacro(mesh, macroEl, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS);
while (elInfo) {
newCoordsFct(elInfo);
elInfo = stack.traverseNext(elInfo);
......
......@@ -44,7 +44,7 @@ namespace AMDiS {
void newCoordsFct(ElInfo *elInfo);
/// Implements RefinementManager::setNewCoords
void setNewCoords();
void setNewCoords(int macroEl = -1);
/** \brief
* gets the elements around the refinement edge with vertices node[0] and
......
......@@ -285,17 +285,26 @@ namespace AMDiS {
}
void RefinementManager3d::setNewCoords()
void RefinementManager3d::setNewCoords(int macroEl)
{
if (refList)
delete refList;
refList = new RCNeighbourList(mesh->getMaxEdgeNeigh());
ElInfo *elInfo;
if (macroEl == -1)
elInfo = stack->traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS |
Mesh::FILL_NEIGH);
else
elInfo = stack->traverseFirstOneMacro(mesh, macroEl, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS |
Mesh::FILL_NEIGH);
ElInfo *elInfo = stack->traverseFirst(mesh, -1,
Mesh::CALL_EVERY_EL_PREORDER |
Mesh::FILL_BOUND | Mesh::FILL_COORDS |
Mesh::FILL_NEIGH);
while (elInfo) {
newCoordsFct(elInfo);
elInfo = stack->traverseNext(elInfo);
......
......@@ -44,7 +44,7 @@ namespace AMDiS {
int newCoordsFct(ElInfo *el_info);
/// Implements RefinementManager::setNewCoords
void setNewCoords();
void setNewCoords(int macroEl = -1);
/** \brief
* Gets the elements around the refinement edge with vertices node[0] and
......
......@@ -56,6 +56,7 @@ namespace AMDiS {
deserialized(false),
writeSerializationFile(false),
repartitioningAllowed(false),
nTimestepsAfterLastRepartitioning(0),
lastMeshChangeIndex(0)
{
FUNCNAME("MeshDistributor::ParalleDomainBase()");
......@@ -547,8 +548,14 @@ namespace AMDiS {
// === The mesh has changed, so check if it is required to repartition the mesh. ===
if (repartitioningAllowed)
repartitionMesh();
nTimestepsAfterLastRepartitioning++;
if (repartitioningAllowed) {
if (nTimestepsAfterLastRepartitioning >= 20) {
repartitionMesh();
nTimestepsAfterLastRepartitioning = 0;
}
}
}
......@@ -965,7 +972,7 @@ namespace AMDiS {
DOFVector<double> tmpa(feSpace, "tmp");
tmpa.set(mpiRank);
VtkWriter::writeFile(tmpa, "before-repartition.vtu");
VtkWriter::writeFile(testVec[0], "before-repartition.vtu");
MSG("USED-SIZE A: %d\n", mesh->getDofAdmin(0).getUsedDofs());
......@@ -985,6 +992,7 @@ namespace AMDiS {
MacroInfo* minfo = mesh->getMacroFileInfo();
TEST_EXIT_DBG(minfo)("No macro file info!\n");
MSG("MARK 1\n");
// === Create map that maps macro element indices to pointers to the ===
// === macro elements. ===
......@@ -994,7 +1002,7 @@ namespace AMDiS {
it != minfo->mel.end(); ++it)
elIndexMap[(*it)->getIndex()] = *it;
MSG("MARK 2\n");
// === Create set of all new macro elements this rank will receive from ===
// === other ranks. ===
......@@ -1014,7 +1022,7 @@ namespace AMDiS {
it != mesh->endOfMacroElements(); ++it)
allMacroEl.insert(*it);
MSG("MARK 3\n");
// === Add new macro elements to mesh. ===
for (std::set<MacroElement*>::iterator it = newMacroEl.begin();
......@@ -1029,7 +1037,7 @@ namespace AMDiS {
mesh->getMacroElements().push_back(mel);
}
MSG("MARK 4\n");
// === Send and receive mesh structure codes. ===
std::map<int, MeshCodeVec> sendCodes;
......@@ -1043,34 +1051,53 @@ namespace AMDiS {
elCode.init(mesh, *elIt);
sendCodes[it->first].push_back(elCode);
std::vector<double> valVec;
elCode.getMeshStructureValues(mesh, *elIt, testVec, valVec);
sendValues[it->first].push_back(valVec);
for (unsigned int i = 0; i < testVec.size(); i++) {
std::vector<double> valVec;
elCode.getMeshStructureValues(mesh, *elIt, testVec[i], valVec);
sendValues[it->first].push_back(valVec);
}
}
}
MSG("MARK 5\n");
StdMpi<MeshCodeVec> stdMpi(mpiComm, true);
stdMpi.send(sendCodes);
stdMpi.recv(partitioner->getRecvElements());
stdMpi.startCommunication<uint64_t>(MPI_UNSIGNED_LONG);
MSG("MARK 6\n");
StdMpi<std::vector<std::vector<double> > > stdMpi2(mpiComm, true);
stdMpi2.send(sendValues);
stdMpi2.recv(partitioner->getRecvElements());
stdMpi2.startCommunication<double>(MPI_DOUBLE);
MSG("MARK 7\n");
// === Adapte the new macro elements due to the received mesh structure codes. ===
for (std::map<int, std::vector<int> >::iterator it = partitioner->getRecvElements().begin();
it != partitioner->getRecvElements().end(); ++it) {
MeshCodeVec &recvCodes = stdMpi.getRecvData()[it->first];
int i = 0;
std::vector<std::vector<double> > &recvValues = stdMpi2.getRecvData()[it->first];
int i = 0;
int j = 0;
TEST_EXIT_DBG(recvCodes.size() == it->second.size())
("Should not happen!\n");
for (std::vector<int>::iterator elIt = it->second.begin();
elIt != it->second.end(); ++elIt)
recvCodes[i++].fitMeshToStructure(mesh, refineManager, false, *elIt);
elIt != it->second.end(); ++elIt) {
MSG("START EL ADAPT\n");
recvCodes[i].fitMeshToStructure(mesh, refineManager, false, *elIt);
MSG("END EL ADAPT\n");
for (unsigned int k = 0; k < testVec.size(); k++) {
MSG("START VAL ADAPT\n");
recvCodes[i].setMeshStructureValues(mesh, *elIt, testVec[k], recvValues[j++]);
MSG("END VAL ADAPT\n");
}
i++;
}
}
MSG("MARK 8\n");
// === Set correct neighbour information on macro elements. ===
for (std::set<MacroElement*>::iterator it = allMacroEl.begin();
......@@ -1085,7 +1112,7 @@ namespace AMDiS {
(*it)->setNeighbour(i, elIndexMap[neighIndex]);
}
}
MSG("MARK 9\n");
// === Delete macros from mesh. ===
......@@ -1101,22 +1128,22 @@ namespace AMDiS {
deleteMacroElements.insert(elIndexMap[*elIt]);
}
}
MSG("MARK 10\n");
mesh->removeMacroElements(deleteMacroElements, feSpace);
if (mpiRank == 0)
debug::writeElementIndexMesh(mesh, "elementIndexxxx.vtu");
// === Remove double DOFs. ===
MSG("MARK 11\n");
MeshManipulation meshManipulation(mesh);
meshManipulation.deleteDoubleDofs(newMacroEl);
MSG("MARK 12\n");
mesh->dofCompress();
DOFVector<double> tmp(feSpace, "tmp");
tmp.set(mpiRank);
VtkWriter::writeFile(tmp, "after-repartition.vtu");
VtkWriter::writeFile(testVec[0], "after-repartition.vtu");
MSG("USED-SIZE B: %d\n", mesh->getDofAdmin(0).getUsedDofs());
......
......@@ -404,7 +404,7 @@ namespace AMDiS {
}
public:
DOFVector<double>* testVec;
std::vector<DOFVector<double>* > testVec;
protected:
///
......@@ -565,6 +565,8 @@ namespace AMDiS {
/// If true, it is possible to repartition the mesh during computations.
bool repartitioningAllowed;
int nTimestepsAfterLastRepartitioning;
/** \brief
* Stores the mesh change index. This is used to recognize changes in the mesh
* structure (e.g. through refinement or coarsening managers).
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment