Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Ein Anmelden über dieses erzeugt ein neues Konto. Das alte Konto ist über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. Logging in via this will create a new account. The old account can be accessed via the "Standard" tab. The administrators

Commit 11c8bbc9 authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Work on Stokes FETI-DP

parent 297ae0f3
......@@ -256,12 +256,47 @@ namespace AMDiS {
vec[it.getDofIndex()] =
stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
}
/// Works in the same way as the function above defined for DOFVectors. Due
/// to performance, this function does not call \ref synchVector for each
/// DOFVector, but instead sends all values of all DOFVectors all at once.
void synchVector(SystemVector &vec, int level = 0);
/// Works quite similar to the function \ref synchVector, but instead the
/// values of subdomain vectors are add along the boundaries.
template<typename T>
void synchAddVector(DOFVector<T> &vec)
{
StdMpi<vector<T> > stdMpi(mpiComm);
const FiniteElemSpace *fe = vec.getFeSpace();
for (DofComm::Iterator it(dofComm.getRecvDofs(), fe);
!it.end(); it.nextRank()) {
vector<T> dofs;
dofs.reserve(it.getDofs().size());
for (; !it.endDofIter(); it.nextDof())
dofs.push_back(vec[it.getDofIndex()]);
stdMpi.send(it.getRank(), dofs);
}
for (DofComm::Iterator it(dofComm.getSendDofs());
!it.end(); it.nextRank())
stdMpi.recv(it.getRank());
stdMpi.startCommunication();
for (DofComm::Iterator it(dofComm.getSendDofs(), fe);
!it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
vec[it.getDofIndex()] +=
stdMpi.getRecvData(it.getRank())[it.getDofCounter()];
synchVector(vec);
}
/// In 3D, a subdomain may not be a valid AMDiS mesh if it contains two
/// parts which are only connected by an edge. In this case, the standard
/// refinement algorithm does not work correctly, as two elements connected
......
......@@ -96,6 +96,8 @@ namespace AMDiS {
Mat matTrans;
MatTranspose(mat, MAT_INITIAL_MATRIX, &matTrans);
bool isSym = true;
if (advancedTest) {
int rowStart, rowEnd;
MatGetOwnershipRange(mat, &rowStart, &rowEnd);
......@@ -105,7 +107,6 @@ namespace AMDiS {
const PetscScalar *vals, *valsTrans;
for (int i = rowStart; i < rowEnd; i++) {
bool isSym = true;
MatGetRow(mat, i, &nCols, &cols, &vals);
MatGetRow(matTrans, i, &nColsTrans, &colsTrans, &valsTrans);
......@@ -140,9 +141,6 @@ namespace AMDiS {
MatRestoreRow(mat, i, &nCols, &cols, &vals);
MatRestoreRow(matTrans, i, &nColsTrans, &colsTrans, &valsTrans);
if (!isSym)
return false;
}
}
......@@ -154,6 +152,6 @@ namespace AMDiS {
MSG("Matrix norm test: %e %e\n", norm1, norm2);
return (norm1 < 1e-10 && norm2 < 1e-10);
return (isSym && norm1 < 1e-10 && norm2 < 1e-10);
}
}
......@@ -112,8 +112,6 @@ namespace AMDiS {
TEST_EXIT(pressureComponent >= 0)
("FETI-DP in Stokes mode, no pressure component defined!\n");
MSG("========= !!!! SUPER WARNING !!! ========\n");
MSG("Use make use of stokes mode which is work in progress! We guarantee for nothing and even less than this!\n");
pressureFeSpace = feSpaces[pressureComponent];
}
......@@ -152,6 +150,11 @@ namespace AMDiS {
{
FUNCNAME("PetscSolverFeti::createDirichletData()");
bool removeDirichletRows = false;
Parameters::get("parallel->feti->remove dirichlet", removeDirichletRows);
if (!removeDirichletRows)
return;
int nComponents = mat.getSize();
for (int i = 0; i < nComponents; i++) {
DOFMatrix* dofMat = mat[i][i];
......@@ -978,7 +981,6 @@ namespace AMDiS {
if (stokesMode) {
DOFVector<double> tmpvec(pressureFeSpace, "tmpvec");
createH2vec(tmpvec);
VtkWriter::writeFile(&tmpvec, "tmpvec.vtu");
interfaceDofMap.createVec(fetiInterfaceLumpedPreconData.h2vec);
DofMap& m = interfaceDofMap[pressureFeSpace].getMap();
......@@ -992,15 +994,12 @@ namespace AMDiS {
VecAssemblyBegin(fetiInterfaceLumpedPreconData.h2vec);
VecAssemblyEnd(fetiInterfaceLumpedPreconData.h2vec);
localDofMap.createVec(fetiInterfaceLumpedPreconData.tmp_vec_b1);
primalDofMap.createVec(fetiInterfaceLumpedPreconData.tmp_primal);
fetiInterfaceLumpedPreconData.subSolver = subdomain;
}
KSPGetPC(ksp_feti, &precon_feti);
PCSetType(precon_feti, PCSHELL);
if (stokesMode) {
KSPCreate(PETSC_COMM_WORLD, &fetiInterfaceLumpedPreconData.ksp_primal);
KSPSetOperators(fetiInterfaceLumpedPreconData.ksp_primal,
subdomain->getMatCoarse(0, 0),
......@@ -1008,7 +1007,12 @@ namespace AMDiS {
SAME_NONZERO_PATTERN);
KSPSetOptionsPrefix(fetiInterfaceLumpedPreconData.ksp_primal, "primal_");
KSPSetFromOptions(fetiInterfaceLumpedPreconData.ksp_primal);
}
KSPGetPC(ksp_feti, &precon_feti);
PCSetType(precon_feti, PCSHELL);
if (stokesMode) {
PCShellSetContext(precon_feti, static_cast<void*>(&fetiInterfaceLumpedPreconData));
PCShellSetApply(precon_feti, petscApplyFetiInterfaceLumpedPrecon);
} else {
......@@ -1129,6 +1133,23 @@ namespace AMDiS {
Vec nullSpaceBasis;
VecCreateNest(mpiCommGlobal, 2, PETSC_NULL, vecArray, &nullSpaceBasis);
int writeFetiSystem = 0;
Parameters::get("parallel->debug->write feti system",
writeFetiSystem);
if (writeFetiSystem) {
Vec vecTmp;
createExplicitVec(nullSpaceBasis, vecTmp);
PetscViewer petscView;
PetscViewerBinaryOpen(PETSC_COMM_WORLD, "nullspace.vec",
FILE_MODE_WRITE, &petscView);
VecView(vecTmp, petscView);
PetscViewerDestroy(&petscView);
VecDestroy(&vecTmp);
MSG("Written FETI-DP null space basis vector!\n");
}
MatNullSpace matNullSpace;
MatNullSpaceCreate(mpiCommGlobal, PETSC_FALSE, 1, &nullSpaceBasis,
&matNullSpace);
......@@ -1264,81 +1285,6 @@ namespace AMDiS {
MatView(mat_schur_primal, petscView);
PetscViewerDestroy(&petscView);
}
int writeFetiMatrix = 0;
Parameters::get("parallel->debug->write feti matrix",
writeFetiMatrix);
if (writeFetiMatrix) {
MSG("Start creating explicit FETI-DP matrix!\n");
Vec unitVector;
Vec resultVector;
Mat fetiMat;
MatCreateAIJ(mpiCommGlobal,
lagrangeMap.getRankDofs(),
lagrangeMap.getRankDofs(),
lagrangeMap.getOverallDofs(),
lagrangeMap.getOverallDofs(),
lagrangeMap.getOverallDofs(),
PETSC_NULL,
lagrangeMap.getOverallDofs(),
PETSC_NULL, &fetiMat);
lagrangeMap.createVec(unitVector);
lagrangeMap.createVec(resultVector);
PetscInt low, high;
VecGetOwnershipRange(unitVector, &low, &high);
int nLocal = high - low;
int nnzCounter = 0;
for (int i = 0; i < lagrangeMap.getOverallDofs(); i++) {
VecSet(unitVector, 0.0);
if (i >= low && i < high)
VecSetValue(unitVector, i, 1.0, INSERT_VALUES);
VecAssemblyBegin(unitVector);
VecAssemblyEnd(unitVector);
MatMult(mat_feti, unitVector, resultVector);
if (fetiPreconditioner != FETI_NONE) {
PCApply(precon_feti, resultVector, unitVector);
VecCopy(unitVector, resultVector);
}
PetscScalar *vals;
VecGetArray(resultVector, &vals);
for (int j = 0; j < nLocal; j++)
if (fabs(vals[j]) > 1e-30) {
MatSetValue(fetiMat, low + j, i, vals[j], INSERT_VALUES);
nnzCounter++;
}
VecRestoreArray(resultVector, &vals);
}
MatAssemblyBegin(fetiMat, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(fetiMat, MAT_FINAL_ASSEMBLY);
VecDestroy(&unitVector);
VecDestroy(&resultVector);
mpi::globalAdd(nnzCounter);
PetscViewer petscView;
PetscViewerBinaryOpen(PETSC_COMM_WORLD, "feti.mat",
FILE_MODE_WRITE, &petscView);
MatView(fetiMat, petscView);
PetscViewerDestroy(&petscView);
MatDestroy(&fetiMat);
MSG("FETI-DP matrix written: %d x %d mat with %d nnz\n",
lagrangeMap.getOverallDofs(), lagrangeMap.getOverallDofs(), nnzCounter);
}
}
......@@ -1525,21 +1471,12 @@ namespace AMDiS {
// === Create matrices for the FETI-DP method. ===
if (printTimings) {
if (printTimings)
MPI::COMM_WORLD.Barrier();
}
double wtime = MPI::Wtime();
subdomain->fillPetscMatrix(mat);
// MSG("MATRIX IS SYM: %d\n", testMatrixSymmetric(subdomain->getMatInterior(), true));
// if (MPI::COMM_WORLD.Get_rank() == 0)
// MatView(subdomain->getMatInterior(), PETSC_VIEWER_STDOUT_SELF);
// AMDiS::finalize();
// exit(0);
if (printTimings) {
MPI::COMM_WORLD.Barrier();
MSG("FETI-DP timing 02: %.5f seconds (creation of interior matrices)\n",
......@@ -1558,23 +1495,18 @@ namespace AMDiS {
// === Create matrices for FETI-DP preconditioner. ===
createPreconditionerMatrix(mat);
// MSG("DUALS MATRIX IS SYM: %d\n", testMatrixSymmetric(mat_duals_duals, true));
// AMDiS::finalize();
// exit(0);
// === Create and fill PETSc matrix for Lagrange constraints. ===
createMatLagrange(feSpaces);
// === If required, run debug tests. ===
dbgMatrix();
// === Create PETSc solver for the Schur complement on primal variables. ===
createSchurPrimalKsp(feSpaces);
// === Create PETSc solver for the FETI-DP operator. ===
createFetiKsp(feSpaces);
// === If required, run debug tests. ===
dbgMatrix();
}
......@@ -1920,6 +1852,8 @@ namespace AMDiS {
FetiTimings::reset();
}
// === Optionally run some debug procedures on the FETI-DP system. ===
debugFeti(vecRhs);
// === Solve with FETI-DP operator. ===
KSPSolve(ksp_feti, vecRhs, vecSol);
......@@ -2055,27 +1989,18 @@ namespace AMDiS {
vec.set(0.0);
DOFVector<int> cvec(vec.getFeSpace(), "cvec");
cvec.set(0.0);
Mesh *mesh = vec.getFeSpace()->getMesh();
TEST_EXIT(mesh->getDim() == 2)("This works only in 2D!\n");
int n0 = vec.getFeSpace()->getAdmin()->getNumberOfPreDofs(VERTEX);
TraverseStack stack;
ElInfo *elInfo =
stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS | Mesh::FILL_DET);
stack.traverseFirst(mesh, -1, Mesh::CALL_LEAF_EL | Mesh::FILL_COORDS);
while (elInfo) {
Element *el = elInfo->getElement();
for (int i = 0; i < 3; i++) {
DegreeOfFreedom dof = el->getDof(i, n0);
vec[dof] += elInfo->getDet() * 0.5;
cvec[dof] += 1;
}
std::vector<double> length(3, 0.0);
double sumLength = 0.0;
/*
for (int i = 0; i < 3; i++) {
int idx0 = el->getVertexOfEdge(i, 0);
int idx1 = el->getVertexOfEdge(i, 1);
......@@ -2085,22 +2010,27 @@ namespace AMDiS {
WorldVector<double> c0 = elInfo->getCoord(idx0);
WorldVector<double> c1 = elInfo->getCoord(idx1);
c0 -= c1;
double length = norm(c0);
vec[dof0] += length;
vec[dof1] += length;
cvec[dof0] += 1;
cvec[dof1] += 1;
length[i] = norm(c0);
sumLength += length[i];
}
sumLength *= 0.5;
double area =
sqrt(sumLength * (sumLength - length[0]) * (sumLength - length[1]) * (sumLength - length[2]));
for (int i = 0; i < 3; i++) {
DegreeOfFreedom d = el->getDof(i, n0);
vec[d] += area;
}
*/
elInfo = stack.traverseNext(elInfo);
}
meshDistributor->synchAddVector(vec);
DOFIterator<double> dofIt(&vec, USED_DOFS);
for (dofIt.reset(); !dofIt.end(); ++dofIt) {
DegreeOfFreedom d = dofIt.getDOFIndex();
vec[d] /= static_cast<double>(cvec[d]);
vec[d] = 1.0 / pow(vec[d], 2.0);
}
}
......@@ -2287,4 +2217,265 @@ namespace AMDiS {
VecDestroy(&nullSpaceBasis);
VecDestroy(&vecSol);
}
void PetscSolverFeti::createExplicitFetiMat(Mat fetiMat,
Mat &explicitMat,
int &nnzCounter)
{
FUNCNAME("PetscSolverFeti::createExplicitFetiMat()");
nnzCounter = 0;
if (stokesMode == false) {
int nRankRows = lagrangeMap.getRankDofs();
int nOverallRows = lagrangeMap.getOverallDofs();
MatCreateAIJ(mpiCommGlobal,
nRankRows, nRankRows, nOverallRows, nOverallRows,
nOverallRows, PETSC_NULL, nOverallRows, PETSC_NULL,
&explicitMat);
Vec unitVector;
Vec resultVector;
lagrangeMap.createVec(unitVector);
lagrangeMap.createVec(resultVector);
PetscInt low, high;
VecGetOwnershipRange(unitVector, &low, &high);
int nLocal = high - low;
int nnzCounter = 0;
for (int i = 0; i < lagrangeMap.getOverallDofs(); i++) {
VecSet(unitVector, 0.0);
if (i >= low && i < high)
VecSetValue(unitVector, i, 1.0, INSERT_VALUES);
VecAssemblyBegin(unitVector);
VecAssemblyEnd(unitVector);
MatMult(fetiMat, unitVector, resultVector);
if (fetiPreconditioner != FETI_NONE) {
PCApply(precon_feti, resultVector, unitVector);
VecCopy(unitVector, resultVector);
}
PetscScalar *vals;
VecGetArray(resultVector, &vals);
for (int j = 0; j < nLocal; j++) {
if (fabs(vals[j]) > 1e-30) {
MatSetValue(explicitMat, low + j, i, vals[j], INSERT_VALUES);
nnzCounter++;
}
}
VecRestoreArray(resultVector, &vals);
}
VecDestroy(&unitVector);
VecDestroy(&resultVector);
} else {
int nRankRows = interfaceDofMap.getRankDofs() + lagrangeMap.getRankDofs();
int nOverallRows =
interfaceDofMap.getOverallDofs() + lagrangeMap.getOverallDofs();
MatCreateAIJ(mpiCommGlobal,
nRankRows, nRankRows, nOverallRows, nOverallRows,
nOverallRows, PETSC_NULL, nOverallRows, PETSC_NULL,
&explicitMat);
Vec unitVector[2];
Vec resultVector[2];
interfaceDofMap.createVec(unitVector[0]);
interfaceDofMap.createVec(resultVector[0]);
lagrangeMap.createVec(unitVector[1]);
lagrangeMap.createVec(resultVector[1]);
Vec unitNestVec, resultNestVec;
VecCreateNest(mpiCommGlobal, 2, PETSC_NULL, unitVector, &unitNestVec);
VecCreateNest(mpiCommGlobal, 2, PETSC_NULL, resultVector, &resultNestVec);
PetscInt lowInterface, highInterface;
VecGetOwnershipRange(unitVector[0], &lowInterface, &highInterface);
int nLocalInterface = highInterface - lowInterface;
PetscInt lowLagrange, highLagrange;
VecGetOwnershipRange(unitVector[1], &lowLagrange, &highLagrange);
int nLocalLagrange = highLagrange - lowLagrange;
VecSet(unitVector[1], 0.0);
for (int i = 0; i < interfaceDofMap.getOverallDofs(); i++) {
VecSet(unitVector[0], 0.0);
if (i >= lowInterface && i < highInterface)
VecSetValue(unitVector[0], i, 1.0, INSERT_VALUES);
VecAssemblyBegin(unitVector[0]);
VecAssemblyEnd(unitVector[0]);
VecAssemblyBegin(unitNestVec);
VecAssemblyEnd(unitNestVec);
MatMult(fetiMat, unitNestVec, resultNestVec);
PetscScalar *vals;
VecGetArray(resultVector[0], &vals);
for (int j = 0; j < nLocalInterface; j++) {
if (fabs(vals[j]) > 1e-30) {
MatSetValue(explicitMat, lowInterface + j, i, vals[j], INSERT_VALUES);
nnzCounter++;
}
}
VecRestoreArray(resultVector[0], &vals);
VecGetArray(resultVector[1], &vals);
for (int j = 0; j < nLocalLagrange; j++) {
if (fabs(vals[j]) > 1e-30) {
MatSetValue(explicitMat,
interfaceDofMap.getOverallDofs() + lowLagrange + j, i,
vals[j], INSERT_VALUES);
nnzCounter++;
}
}
VecRestoreArray(resultVector[1], &vals);
}
VecSet(unitVector[0], 0.0);
for (int i = 0; i < lagrangeMap.getOverallDofs(); i++) {
VecSet(unitVector[1], 0.0);
if (i >= lowLagrange && i < highLagrange)
VecSetValue(unitVector[1], i, 1.0, INSERT_VALUES);
VecAssemblyBegin(unitVector[1]);
VecAssemblyEnd(unitVector[1]);
VecAssemblyBegin(unitNestVec);
VecAssemblyEnd(unitNestVec);
MatMult(fetiMat, unitNestVec, resultNestVec);
PetscScalar *vals;
VecGetArray(resultVector[0], &vals);
for (int j = 0; j < nLocalInterface; j++) {
if (fabs(vals[j]) > 1e-30) {
MatSetValue(explicitMat,
lowInterface + j,
interfaceDofMap.getOverallDofs() + i,
vals[j], INSERT_VALUES);
nnzCounter++;
}
}
VecRestoreArray(resultVector[0], &vals);
VecGetArray(resultVector[1], &vals);
for (int j = 0; j < nLocalLagrange; j++) {
if (fabs(vals[j]) > 1e-30) {
MatSetValue(explicitMat,
interfaceDofMap.getOverallDofs() + lowLagrange + j,
interfaceDofMap.getOverallDofs() + i,
vals[j], INSERT_VALUES);
nnzCounter++;
}
}
VecRestoreArray(resultVector[1], &vals);
}
VecDestroy(&unitVector[0]);
VecDestroy(&unitVector[1]);
VecDestroy(&resultVector[0]);
VecDestroy(&resultVector[1]);
VecDestroy(&unitNestVec);
VecDestroy(&resultNestVec);
}
MatAssemblyBegin(explicitMat, MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(explicitMat, MAT_FINAL_ASSEMBLY);
mpi::globalAdd(nnzCounter);
}
void PetscSolverFeti::createExplicitVec(Vec nestedVec, Vec &explicitVec)
{
FUNCNAME("PetscSolverFeti::createExplicitVec()");
int nNested = 0;
VecNestGetSize(nestedVec, &nNested);
TEST_EXIT_DBG(nNested == 2)
("Only supported for 2-nest vectors, not for %d-nest vectors!\n", nNested);
Vec v0, v1;
VecNestGetSubVec(nestedVec, 0, &v0);
VecNestGetSubVec(nestedVec, 1, &v1);
int s0, s1;
VecGetSize(v0, &s0);
VecGetSize(v1, &s1);
int l0, l1;
VecGetLocalSize(v0, &l0);
VecGetLocalSize(v1, &l1);
int rStart0, rStart1;
VecGetOwnershipRange(v0, &rStart0, PETSC_NULL);
VecGetOwnershipRange(v1, &rStart1, PETSC_NULL);
VecCreateMPI(mpiCommGlobal, l0 + l1, s0 + s1, &explicitVec);
double *val;
VecGetArray(v0, &val);
for (int i = 0; i < l0; i++)
VecSetValue(explicitVec, rStart0 + i, val[i], INSERT_VALUES);
VecRestoreArray(v0, &val);
VecGetArray(v1, &val);
for (int i = 0; i < l1; i++)
VecSetValue(explicitVec, s0 + rStart1 + i, val[i], INSERT_VALUES);
VecRestoreArray(v1, &val);
VecAssemblyBegin(explicitVec);
VecAssemblyEnd(explicitVec);
}
void PetscSolverFeti::debugFeti(Vec dbgRhsVec)
{
FUNCNAME("PetscSolverFeti:::debugFeti()");
int writeFetiSystem = 0;
Parameters::get("parallel->debug->write feti system",
writeFetiSystem);
if (!writeFetiSystem)