Commit f94e0e7c authored by Thomas Witkowski's avatar Thomas Witkowski

Changed coarse space creation for FETI-DP method.

parent 70d8921a
...@@ -22,7 +22,7 @@ namespace AMDiS { ...@@ -22,7 +22,7 @@ namespace AMDiS {
using namespace std; using namespace std;
#if defined(HAVE_PARALLEL_DOMAIN_AMDIS) && defined(HAVE_PARALLEL_MTL4) #if defined(HAVE_PARALLEL_DOMAIN_AMDIS) && defined(HAVE_PARALLEL_MTL4)
mtl::par::environment* mtl_environment=NULL; mtl::par::environment* mtl_environment = NULL;
#endif #endif
void init(int argc, char **argv, std::string initFileName) void init(int argc, char **argv, std::string initFileName)
...@@ -37,8 +37,6 @@ namespace AMDiS { ...@@ -37,8 +37,6 @@ namespace AMDiS {
mpi::startRand(); mpi::startRand();
#endif #endif
Parameters::get("parallel->log main rank", Msg::outputMainRank);
#ifdef HAVE_ZOLTAN #ifdef HAVE_ZOLTAN
float zoltanVersion = 0.0; float zoltanVersion = 0.0;
Zoltan_Initialize(argc, argv, &zoltanVersion); Zoltan_Initialize(argc, argv, &zoltanVersion);
...@@ -98,6 +96,8 @@ namespace AMDiS { ...@@ -98,6 +96,8 @@ namespace AMDiS {
Parameters::init(initFileName); Parameters::init(initFileName);
} }
Parameters::get("parallel->log main rank", Msg::outputMainRank);
// reset parameters from command line // reset parameters from command line
bool ignoreCommandline = false; bool ignoreCommandline = false;
Parameters::get("ignore commandline options", ignoreCommandline); Parameters::get("ignore commandline options", ignoreCommandline);
......
...@@ -734,51 +734,78 @@ namespace AMDiS { ...@@ -734,51 +734,78 @@ namespace AMDiS {
} }
std::set<BoundaryObject> PetscSolverFeti::getAugmentedLagrange() vector<vector<BoundaryObject> > PetscSolverFeti::getCoarseEdges()
{ {
FUNCNAME("PetscSolverFeti::getAugmentedLagrange()"); FUNCNAME("PetscSolverFeti::getAugmentedLagrange()");
int nEdges = 0;
int nFaces = 0;
InteriorBoundary &intBound = meshDistributor->getIntBoundary(); InteriorBoundary &intBound = meshDistributor->getIntBoundary();
std::set<BoundaryObject> allEdges; std::set<BoundaryObject> allEdges;
for (InteriorBoundary::iterator it(intBound.getOwn()); !it.end(); ++it) { for (InteriorBoundary::iterator it(intBound.getOwn()); !it.end(); ++it)
if ((it->rankObj.subObj == FACE || if (it->rankObj.subObj == EDGE &&
(it->rankObj.subObj == EDGE && testWirebasketEdge(it->rankObj, feSpaces[0]) &&
testWirebasketEdge(it->rankObj, feSpaces[0]))) && allEdges.count(it->rankObj) == 0)
allEdges.count(it->rankObj) == 0) { allEdges.insert(it->rankObj);
bool dirichletOnlyEdge = true;
int nEmptyEdges = 0;
DofContainer edgeDofs; vector<vector<BoundaryObject> > data;
it->rankObj.el->getAllDofs(feSpaces[0], it->rankObj, edgeDofs); for (std::set<BoundaryObject>::iterator it = allEdges.begin();
it != allEdges.end(); ++it) {
for (DofContainer::iterator dit = edgeDofs.begin(); DofContainer edgeDofs;
dit != edgeDofs.end(); ++dit) { it->el->getAllDofs(feSpaces[0], *it, edgeDofs);
if (dirichletRows[0].count(**dit) == 0) { if (edgeDofs.size() == 0) {
dirichletOnlyEdge = false; nEmptyEdges++;
break; } else {
} vector<BoundaryObject> oneBoundary;
} oneBoundary.push_back(*it);
data.push_back(oneBoundary);
}
}
if (!dirichletOnlyEdge) { int nEdges = allEdges.size();
if (it->rankObj.subObj == EDGE) mpi::globalAdd(nEdges);
nEdges++; mpi::globalAdd(nEmptyEdges);
if (it->rankObj.subObj == FACE)
nFaces++;
allEdges.insert(it->rankObj); MSG("Coarse space edges: %d (empty: %d)\n", nEdges, nEmptyEdges);
}
return data;
}
vector<vector<BoundaryObject> > PetscSolverFeti::getCoarseFaces()
{
FUNCNAME("PetscSolverFeti::getAugmentedLagrange()");
InteriorBoundary &intBound = meshDistributor->getIntBoundary();
map<int, std::set<BoundaryObject> > allFaces;
for (InteriorBoundary::iterator it(intBound.getOwn()); !it.end(); ++it)
if (it->rankObj.subObj == FACE)
allFaces[it.getRank()].insert(it->rankObj);
int nEmptyFaces = 0;
vector<vector<BoundaryObject> > data;
for (map<int, std::set<BoundaryObject> >::iterator it = allFaces.begin();
it != allFaces.end(); ++it) {
vector<BoundaryObject> oneBoundary;
for (std::set<BoundaryObject>::iterator bIt = it->second.begin();
bIt != it->second.end(); ++bIt) {
DofContainer faceDofs;
bIt->el->getAllDofs(feSpaces[0], *bIt, faceDofs);
if (faceDofs.size())
oneBoundary.push_back(*bIt);
} }
if (oneBoundary.size())
data.push_back(oneBoundary);
else
nEmptyFaces++;
} }
mpi::globalAdd(nEdges); int nFaces = allFaces.size();
mpi::globalAdd(nFaces); mpi::globalAdd(nFaces);
mpi::globalAdd(nEmptyFaces);
MSG("NEDGES = %d NFACES = %d\n", nEdges, nFaces); MSG("Coarse space faces: %d (empty: %d)\n", nFaces, nEmptyFaces);
return allEdges; return data;
} }
...@@ -791,8 +818,11 @@ namespace AMDiS { ...@@ -791,8 +818,11 @@ namespace AMDiS {
double wtime = MPI::Wtime(); double wtime = MPI::Wtime();
std::set<BoundaryObject> allEdges = getAugmentedLagrange(); vector<vector<BoundaryObject> > allEdges = getCoarseEdges();
nRankEdges = allEdges.size(); vector<vector<BoundaryObject> > allFaces = getCoarseFaces();
allEdges.insert(allEdges.end(), allFaces.begin(), allFaces.end());
nRankEdges = allEdges.size();
int rStartEdges = 0; int rStartEdges = 0;
mpi::getDofNumbering(mpiCommGlobal, nRankEdges, rStartEdges, nOverallEdges); mpi::getDofNumbering(mpiCommGlobal, nRankEdges, rStartEdges, nOverallEdges);
...@@ -810,27 +840,29 @@ namespace AMDiS { ...@@ -810,27 +840,29 @@ namespace AMDiS {
MatSetOption(mat_augmented_lagrange, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); MatSetOption(mat_augmented_lagrange, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
int rowCounter = rStartEdges; int rowCounter = rStartEdges;
for (std::set<BoundaryObject>::iterator edgeIt = allEdges.begin(); for (vector<vector<BoundaryObject> >::iterator it = allEdges.begin();
edgeIt != allEdges.end(); ++edgeIt) { it != allEdges.end(); ++it) {
for (int component = 0; component < componentSpaces.size(); component++) { for (int component = 0; component < componentSpaces.size(); component++) {
DofContainer edgeDofs; for (vector<BoundaryObject>::iterator edgeIt = it->begin();
edgeIt->el->getAllDofs(componentSpaces[component], *edgeIt, edgeDofs); edgeIt != it->end(); ++edgeIt) {
for (DofContainer::iterator it = edgeDofs.begin();
it != edgeDofs.end(); ++it) {
TEST_EXIT_DBG(isPrimal(component, **it) == false)
("Should not be primal!\n");
if (dirichletRows[component].count(**it)) DofContainer edgeDofs;
continue; edgeIt->el->getAllDofs(componentSpaces[component], *edgeIt, edgeDofs);
TEST_EXIT(edgeDofs.size())("Should not happen!\n");
int col = lagrangeMap.getMatIndex(component, **it); for (DofContainer::iterator it = edgeDofs.begin();
double value = 1.0; it != edgeDofs.end(); ++it) {
MatSetValue(mat_augmented_lagrange, rowCounter, col, value, INSERT_VALUES); TEST_EXIT(isPrimal(component, **it) == false)
("Should not be primal!\n");
int col = lagrangeMap.getMatIndex(component, **it);
double value = 1.0;
MatSetValue(mat_augmented_lagrange, rowCounter, col, value, INSERT_VALUES);
}
} }
rowCounter++;
rowCounter++; }
}
} }
MatAssemblyBegin(mat_augmented_lagrange, MAT_FINAL_ASSEMBLY); MatAssemblyBegin(mat_augmented_lagrange, MAT_FINAL_ASSEMBLY);
......
...@@ -131,7 +131,9 @@ namespace AMDiS { ...@@ -131,7 +131,9 @@ namespace AMDiS {
/// to \ref mat_lagrange. /// to \ref mat_lagrange.
void createMatLagrange(); void createMatLagrange();
std::set<BoundaryObject> getAugmentedLagrange(); vector<vector<BoundaryObject> > getCoarseEdges();
vector<vector<BoundaryObject> > getCoarseFaces();
void createMatAugmentedLagrange(); void createMatAugmentedLagrange();
......
...@@ -641,23 +641,26 @@ namespace AMDiS { ...@@ -641,23 +641,26 @@ namespace AMDiS {
StdMpi<vector<double> > stdMpi(MPI::COMM_WORLD); StdMpi<vector<double> > stdMpi(MPI::COMM_WORLD);
vector<double> sendData; vector<double> sendData;
std::set<BoundaryObject> coarseSpace = feti.getAugmentedLagrange(); vector<vector<BoundaryObject> > coarseSpace = feti.getCoarseFaces();
for (std::set<BoundaryObject>::iterator it = coarseSpace.begin(); for (vector<vector<BoundaryObject> >::iterator it = coarseSpace.begin();
it != coarseSpace.end(); ++it) { it != coarseSpace.end(); ++it) {
if (it->subObj == FACE) { for (vector<BoundaryObject>::iterator edgeIt = it->begin();
DofFace face = it->el->getFace(it->ithObj); edgeIt != it->end(); ++edgeIt) {
WorldVector<double> c0 = coords[face.get<0>()]; if (edgeIt->subObj == FACE) {
WorldVector<double> c1 = coords[face.get<1>()]; DofFace face = edgeIt->el->getFace(edgeIt->ithObj);
WorldVector<double> c2 = coords[face.get<2>()]; WorldVector<double> c0 = coords[face.get<0>()];
sendData.push_back(c0[0]); WorldVector<double> c1 = coords[face.get<1>()];
sendData.push_back(c0[1]); WorldVector<double> c2 = coords[face.get<2>()];
sendData.push_back(c0[2]); sendData.push_back(c0[0]);
sendData.push_back(c1[0]); sendData.push_back(c0[1]);
sendData.push_back(c1[1]); sendData.push_back(c0[2]);
sendData.push_back(c1[2]); sendData.push_back(c1[0]);
sendData.push_back(c2[0]); sendData.push_back(c1[1]);
sendData.push_back(c2[1]); sendData.push_back(c1[2]);
sendData.push_back(c2[2]); sendData.push_back(c2[0]);
sendData.push_back(c2[1]);
sendData.push_back(c2[2]);
}
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment