Skip to content

GitLab

Liebe Gitlab-Nutzer, lieber Gitlab-Nutzer, es ist nun möglich sich mittels des ZIH-Logins/LDAP an unserem Dienst anzumelden. Die Konto der externen Nutzer:innen sind über den Reiter "Standard" erreichbar. Die Administratoren

Dear Gitlab user, it is now possible to log in to our service using the ZIH login/LDAP. The accounts of external users can be accessed via the "Standard" tab. The administrators

Commit dc9956be authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files

Go to juropa.

parent ccd7eecd
Hide whitespace changes
Inline Side-by-side
Showing with 154 additions and 84 deletions
AMDiS/src/parallel/MatrixNnzStructure.cc
View file @ dc9956be
......@@ -48,6 +48,8 @@ namespace AMDiS {
int nRankCols = colDofMap.getRankDofs();
int rankStartColIndex = colDofMap.getStartDofs();
MSG("RANK DOFS FOR NNZ = %d %d\n", nRankRows, (!localMatrix ? nRankRows : -1));
create(nRankRows, (!localMatrix ? nRankRows : -1));
using mtl::tag::row; using mtl::tag::nz; using mtl::begin; using mtl::end;
......@@ -115,7 +117,10 @@ namespace AMDiS {
const FiniteElemSpace *rowFeSpace = dofMat->getRowFeSpace();
const FiniteElemSpace *colFeSpace = dofMat->getColFeSpace();
if (rowDofMap.isDefinedFor(rowFeSpace) == false ||
colDofMap.isDefinedFor(colFeSpace) == false)
continue;
// === Prepare MTL4 iterators for the current DOF matrix. ===
......@@ -246,9 +251,11 @@ namespace AMDiS {
if (colDofMap[colFeSpace].find(col(*icursor), colDofIndex) == false)
continue;
MSG("GET COL INDEX: %d\n", colComp);
int petscColIdx = (colDofMap.isMatIndexFromGlobal() ?
colDofMap.getMatIndex(colComp, colDofIndex.global) :
colDofMap.getMatIndex(colComp, col(*icursor)));
MSG("GOT\n");
sendMatrixEntry[sendToRank].
push_back(make_pair(petscRowIdx, petscColIdx));
......
AMDiS/src/parallel/ParallelCoarseSpaceMatVec.cc
View file @ dc9956be
......@@ -100,12 +100,15 @@ namespace AMDiS {
TEST_EXIT_DBG(found)("Should not happen!\n");
}
for (int i = 0; i < componentIthCoarseMap.size(); i++)
MSG("COMP %d -> %d\n", i, componentIthCoarseMap[i]);
}
void ParallelCoarseSpaceMatVec::createMatVec(Matrix<DOFMatrix*>& seqMat)
{
FUNCNAME("ParallelCoarseSpaceMatVec::create()");
FUNCNAME("ParallelCoarseSpaceMatVec::createMatVec()");
// === Prepare coarse space information and generate the correct number ===
// === of empty PETSc matrix and vector objects. ===
......@@ -135,6 +138,7 @@ namespace AMDiS {
nnz[i][j].clear();
}
MSG("CREATE MAT NNZ 0 - 0\n");
nnz[0][0].create(seqMat, mpiCommGlobal, *interiorMap,
(coarseSpaceMap.size() == 0 ? &(meshDistributor->getPeriodicMap()) : NULL),
meshDistributor->getElementObjectDb(),
......@@ -145,18 +149,15 @@ namespace AMDiS {
if (i == 0 && j == 0)
continue;
if (i == j) {
nnz[i][j].create(seqMat, mpiCommGlobal, *(coarseSpaceMap[i - 1]), NULL,
meshDistributor->getElementObjectDb());
} else {
ParallelDofMapping &rowMap =
(i == 0 ? *interiorMap : *(coarseSpaceMap[i - 1]));
ParallelDofMapping &colMap =
(j == 0 ? *interiorMap : *(coarseSpaceMap[j - 1]));
nnz[i][j].create(seqMat, mpiCommGlobal, rowMap, colMap, NULL,
meshDistributor->getElementObjectDb());
}
ParallelDofMapping &rowMap =
(i == 0 ? *interiorMap : *(uniqueCoarseMap[i - 1]));
ParallelDofMapping &colMap =
(j == 0 ? *interiorMap : *(uniqueCoarseMap[j - 1]));
MSG("CREATE MAT NNZ %d - %d WITH ROWS %d\n", i, j, rowMap.getRankDofs());
nnz[i][j].create(seqMat, mpiCommGlobal, rowMap, colMap, NULL,
meshDistributor->getElementObjectDb());
}
}
}
......@@ -165,60 +166,81 @@ namespace AMDiS {
// === Create PETSc matrices and PETSc vectors with the computed ===
// === nnz data structure. ===
int nRankRows = interiorMap->getRankDofs();
int nOverallRows = interiorMap->getOverallDofs();
int nRankInteriorRows = interiorMap->getRankDofs();
int nOverallInteriorRows = interiorMap->getOverallDofs();
MSG("CREATE INTERIOR MAT!\n");
if (localMatrix) {
MatCreateSeqAIJ(mpiCommLocal, nRankRows, nRankRows,
MatCreateSeqAIJ(mpiCommLocal, nRankInteriorRows, nRankInteriorRows,
0, nnz[0][0].dnnz,
&mat[0][0]);
MatSetOption(mat[0][0], MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
} else {
MatCreateAIJ(mpiCommGlobal, nRankRows, nRankRows,
nOverallRows, nOverallRows,
MatCreateAIJ(mpiCommGlobal, nRankInteriorRows, nRankInteriorRows,
nOverallInteriorRows, nOverallInteriorRows,
0, nnz[0][0].dnnz, 0, nnz[0][0].onnz,
&mat[0][0]);
MatSetOption(mat[0][0], MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
}
VecCreateMPI(mpiCommGlobal, nRankRows, nOverallRows, &vecSol[0]);
VecCreateMPI(mpiCommGlobal, nRankRows, nOverallRows, &vecRhs[0]);
VecCreateMPI(mpiCommGlobal, nRankInteriorRows, nOverallInteriorRows, &vecSol[0]);
VecCreateMPI(mpiCommGlobal, nRankInteriorRows, nOverallInteriorRows, &vecRhs[0]);
MSG("DONE!\n");
int nCoarseMap = uniqueCoarseMap.size();
for (int i = 0; i < nCoarseMap; i++) {
ParallelDofMapping* cMap = uniqueCoarseMap[i];
int nRowsRankCoarse = cMap->getRankDofs();
int nRowsOverallCoarse = cMap->getOverallDofs();
int nRankCoarseRows = cMap->getRankDofs();
int nOverallCoarseRows = cMap->getOverallDofs();
MSG("CREATE MAT COARSE %d OF SIZE %d %d\n", i + 1, nRankCoarseRows, nOverallCoarseRows);
MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankCoarse,
nRowsOverallCoarse, nRowsOverallCoarse,
nRankCoarseRows, nRankCoarseRows,
nOverallCoarseRows, nOverallCoarseRows,
0, nnz[i + 1][i + 1].dnnz, 0, nnz[i + 1][i + 1].onnz,
&mat[i + 1][i + 1]);
VecCreateMPI(mpiCommGlobal, nRowsRankCoarse, nRowsOverallCoarse,
MSG("DONE!\n");
VecCreateMPI(mpiCommGlobal, nRankCoarseRows, nOverallCoarseRows,
&vecSol[i + 1]);
VecCreateMPI(mpiCommGlobal, nRowsRankCoarse, nRowsOverallCoarse,
VecCreateMPI(mpiCommGlobal, nRankCoarseRows, nOverallCoarseRows,
&vecRhs[i + 1]);
}
MSG("NCOARSEMAP = %d\n", nCoarseMap);
for (int i = 0; i < nCoarseMap + 1; i++) {
for (int j = 0; j < nCoarseMap + 1; j++) {
int nRowsRankMat = (j == 0 ? nRankRows : uniqueCoarseMap[j - 1]->getRankDofs());
int nRowsOverallMat = (j == 0 ? nOverallRows : uniqueCoarseMap[j - 1]->getOverallDofs());
if (i == j)
continue;
MSG("CHECK FOR %d %d\n", i, j);
int nRowsRankMat = (i == 0 ? nRankInteriorRows : uniqueCoarseMap[i - 1]->getRankDofs());
int nRowsOverallMat = (i == 0 ? nOverallInteriorRows : uniqueCoarseMap[i - 1]->getOverallDofs());
int nColsRankMat = (j == 0 ? nRankInteriorRows : uniqueCoarseMap[j - 1]->getRankDofs());
int nColsOverallMat = (j == 0 ? nOverallInteriorRows : uniqueCoarseMap[j - 1]->getOverallDofs());
MSG("CREATE-A MAT %d %d\n", i, j);
MatCreateAIJ(mpiCommGlobal,
nRowsRankCoarse, nRowsRankMat,
nRowsOverallCoarse, nRowsOverallMat,
0, nnz[i + 1][j].dnnz, 0, nnz[i + 1][j].onnz,
&mat[i + 1][j]);
nRowsRankMat, nColsRankMat,
nRowsOverallMat, nColsOverallMat,
0, nnz[i][j].dnnz, 0, nnz[i][j].onnz,
&mat[i][j]);
MSG("DONE!\n");
MSG("CREATE-B MAT %d %d WITH SIZE ... \n", j, i);
MatCreateAIJ(mpiCommGlobal,
nRowsRankMat, nRowsRankCoarse,
nRowsOverallMat, nRowsOverallCoarse,
0, nnz[j][i + 1].dnnz, 0, nnz[j][i + 1].onnz,
&mat[j][i + 1]);
nColsRankMat, nRowsRankMat,
nColsOverallMat, nRowsOverallMat,
0, nnz[j][i].dnnz, 0, nnz[j][i].onnz,
&mat[j][i]);
MSG("DONE!\n");
}
}
}
......
AMDiS/src/parallel/ParallelDofMapping.h
View file @ dc9956be
......@@ -388,6 +388,11 @@ namespace AMDiS {
return data.find(feSpace)->second;
}
inline bool isDefinedFor(const FiniteElemSpace* feSpace) const
{
return static_cast<bool>(data.count(feSpace));
}
/// Returns the number of solution components the mapping is defined on.
inline int getNumberOfComponents() const
{
......
AMDiS/src/parallel/PetscSolverFeti.cc
View file @ dc9956be
......@@ -226,7 +226,8 @@ namespace AMDiS {
meshLevel(0),
rStartInterior(0),
nGlobalOverallInterior(0),
printTimings(false)
printTimings(false),
enableStokesMode(false)
{
FUNCNAME("PetscSolverFeti::PetscSolverFeti()");
......@@ -269,6 +270,15 @@ namespace AMDiS {
MeshLevelData& levelData = meshDistributor->getMeshLevelData();
vector<const FiniteElemSpace*>& uniqueFe = meshDistributor->getFeSpaces();
enableStokesMode = false;
Parameters::get("parallel->stokes mode", enableStokesMode);
if (enableStokesMode) {
MSG("========= !!!! SUPER WARNING !!! ========\n");
MSG("Use make use of stokes mode which is work in progress! We guarantee for nothing and even less than this!\n");
fullInterface = feSpaces[2];
}
if (subdomain == NULL) {
subdomain = new PetscSolverGlobalMatrix();
......@@ -289,7 +299,8 @@ namespace AMDiS {
lagrangeMap.init(levelData, feSpaces, uniqueFe);
if (fullInterface != NULL)
interfaceDofMap.init(levelData, fullInterface);
interfaceDofMap.init(levelData, feSpaces, uniqueFe);
// interfaceDofMap.init(levelData, fullInterface);
if (fetiPreconditioner != FETI_NONE) {
TEST_EXIT(meshLevel == 0)
......@@ -396,25 +407,39 @@ namespace AMDiS {
const FiniteElemSpace *feSpace = meshDistributor->getFeSpace(i);
MSG("FETI-DP data for %d-ith FE space:\n", i);
MSG(" nRankPrimals = %d nOverallPrimals = %d\n",
primalDofMap[feSpace].nRankDofs,
primalDofMap[feSpace].nOverallDofs);
MSG(" nRankDuals = %d nOverallDuals = %d\n",
dualDofMap[feSpace].nRankDofs,
dualDofMap[feSpace].nOverallDofs);
MSG(" nRankLagrange = %d nOverallLagrange = %d\n",
lagrangeMap[feSpace].nRankDofs,
lagrangeMap[feSpace].nOverallDofs);
if (feSpace == fullInterface) {
MSG(" nRankInterface = %d nOverallInterface = %d\n",
interfaceDofMap[feSpace].nRankDofs,
interfaceDofMap[feSpace].nOverallDofs);
} else {
MSG(" nRankPrimals = %d nOverallPrimals = %d\n",
primalDofMap[feSpace].nRankDofs,
primalDofMap[feSpace].nOverallDofs);
MSG(" nRankDuals = %d nOverallDuals = %d\n",
dualDofMap[feSpace].nRankDofs,
dualDofMap[feSpace].nOverallDofs);
MSG(" nRankLagrange = %d nOverallLagrange = %d\n",
lagrangeMap[feSpace].nRankDofs,
lagrangeMap[feSpace].nOverallDofs);
TEST_EXIT_DBG(localDofMap[feSpace].size() + primalDofMap[feSpace].size() ==
static_cast<unsigned int>(feSpace->getAdmin()->getUsedDofs()))
("Should not happen!\n");
TEST_EXIT_DBG(localDofMap[feSpace].size() + primalDofMap[feSpace].size() ==
static_cast<unsigned int>(feSpace->getAdmin()->getUsedDofs()))
("Should not happen!\n");
}
}
subdomain->setDofMapping(&localDofMap);
subdomain->setCoarseSpaceDofMapping(&primalDofMap);
if (enableStokesMode) {
MSG("WARNING: MAKE THIS MORE GENERAL!!!!!\n");
subdomain->setCoarseSpaceDofMapping(&primalDofMap, 0);
subdomain->setCoarseSpaceDofMapping(&primalDofMap, 1);
subdomain->setCoarseSpaceDofMapping(&interfaceDofMap, 2);
} else {
subdomain->setCoarseSpaceDofMapping(&primalDofMap);
}
if (printTimings) {
MPI::COMM_WORLD.Barrier();
......
AMDiS/src/parallel/PetscSolverFeti.h
View file @ dc9956be
......@@ -275,6 +275,7 @@ namespace AMDiS {
FetiLumpedPreconData fetiLumpedPreconData;
/// Matrices for Dirichlet preconditioner.
Mat mat_interior_interior, mat_duals_duals, mat_interior_duals, mat_duals_interior;
KSP ksp_interior;
......@@ -292,6 +293,8 @@ namespace AMDiS {
const FiniteElemSpace* fullInterface;
bool printTimings;
bool enableStokesMode;
};
......
AMDiS/src/parallel/PetscSolverGlobalMatrix.cc
View file @ dc9956be
......@@ -129,22 +129,25 @@ namespace AMDiS {
// === the global PETSc matrices. ===
int nComponents = seqMat->getSize();
for (int i = 0; i < nComponents; i++) {
for (int j = 0; j < nComponents; j++) {
if (!(*seqMat)[i][j])
for (int rowComponent = 0; rowComponent < nComponents; rowComponent++) {
for (int colComponent = 0; colComponent < nComponents; colComponent++) {
DOFMatrix* dofMat = (*seqMat)[rowComponent][colComponent];
if (!dofMat)
continue;
ParallelDofMapping *rowCoarseSpace = coarseSpaceMap[i];
ParallelDofMapping *colCoarseSpace = coarseSpaceMap[j];
ParallelDofMapping *rowCoarseSpace = coarseSpaceMap[rowComponent];
ParallelDofMapping *colCoarseSpace = coarseSpaceMap[colComponent];
traits::col<Matrix>::type col((*seqMat)[i][j]->getBaseMatrix());
traits::const_value<Matrix>::type value((*seqMat)[i][j]->getBaseMatrix());
traits::col<Matrix>::type col(dofMat->getBaseMatrix());
traits::const_value<Matrix>::type value(dofMat->getBaseMatrix());
// Traverse all rows.
for (cursor_type cursor = begin<row>((*seqMat)[i][j]->getBaseMatrix()),
cend = end<row>((*seqMat)[i][j]->getBaseMatrix()); cursor != cend; ++cursor) {
for (cursor_type cursor = begin<row>(dofMat->getBaseMatrix()),
cend = end<row>(dofMat->getBaseMatrix()); cursor != cend; ++cursor) {
bool isRowCoarse = isCoarseSpace(i, feSpaces[i], *cursor);
bool isRowCoarse =
isCoarseSpace(rowComponent, feSpaces[colComponent], *cursor);
cols.clear();
colsOther.clear();
......@@ -155,7 +158,8 @@ namespace AMDiS {
for (icursor_type icursor = begin<nz>(cursor), icend = end<nz>(cursor);
icursor != icend; ++icursor) {
bool isColCoarse = isCoarseSpace(j, feSpaces[j], col(*icursor));
bool isColCoarse =
isCoarseSpace(colComponent, feSpaces[colComponent], col(*icursor));
if (isColCoarse) {
if (isRowCoarse) {
......@@ -180,56 +184,60 @@ namespace AMDiS {
// === Set matrix values. ===
if (isRowCoarse) {
int rowIndex = rowCoarseSpace->getMatIndex(i, *cursor);
for (unsigned int k = 0; k < cols.size(); k++)
cols[k] = colCoarseSpace->getMatIndex(j, cols[k]);
int rowIndex = rowCoarseSpace->getMatIndex(rowComponent, *cursor);
for (unsigned int i = 0; i < cols.size(); i++)
cols[i] = colCoarseSpace->getMatIndex(colComponent, cols[i]);
// matCoarseCoarse
MatSetValues(getMatCoarseByComponent(i),
MatSetValues(getMatCoarseByComponent(rowComponent),
1, &rowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES);
if (colsOther.size()) {
if (subdomainLevel == 0) {
for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] = interiorMap->getMatIndex(j, colsOther[k]);
for (unsigned int i = 0; i < colsOther.size(); i++)
colsOther[i] =
interiorMap->getMatIndex(colComponent, colsOther[i]);
} else {
for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] =
interiorMap->getMatIndex(j, colsOther[k]) + rStartInterior;
for (unsigned int i = 0; i < colsOther.size(); i++)
colsOther[i] =
interiorMap->getMatIndex(colComponent, colsOther[i]) +
rStartInterior;
}
// matCoarseInt
MatSetValues(getMatCoarseInteriorByComponent(i),
MatSetValues(getMatCoarseInteriorByComponent(rowComponent),
1, &rowIndex, colsOther.size(),
&(colsOther[0]), &(valuesOther[0]), ADD_VALUES);
}
} else {
int localRowIndex =
(subdomainLevel == 0 ? interiorMap->getLocalMatIndex(i, *cursor) :
interiorMap->getMatIndex(i, *cursor));
(subdomainLevel == 0 ?
interiorMap->getLocalMatIndex(rowComponent, *cursor) :
interiorMap->getMatIndex(rowComponent, *cursor));
for (unsigned int k = 0; k < cols.size(); k++) {
for (unsigned int i = 0; i < cols.size(); i++) {
if (subdomainLevel == 0)
cols[k] = interiorMap->getLocalMatIndex(j, cols[k]);
cols[i] = interiorMap->getLocalMatIndex(colComponent, cols[i]);
else
cols[k] = interiorMap->getMatIndex(j, cols[k]);
cols[i] = interiorMap->getMatIndex(colComponent, cols[i]);
}
MatSetValues(getMatInterior(), 1, &localRowIndex, cols.size(),
&(cols[0]), &(values[0]), ADD_VALUES);
if (colsOther.size()) {
int globalRowIndex = interiorMap->getMatIndex(i, *cursor);
int globalRowIndex = interiorMap->getMatIndex(rowComponent, *cursor);
if (subdomainLevel != 0)
globalRowIndex += rStartInterior;
for (unsigned int k = 0; k < colsOther.size(); k++)
colsOther[k] = colCoarseSpace->getMatIndex(j, colsOther[k]);
for (unsigned int i = 0; i < colsOther.size(); i++)
colsOther[i] =
colCoarseSpace->getMatIndex(colComponent, colsOther[i]);
// matIntCoarse
MatSetValues(getMatInteriorCoarseByComponent(i),
MatSetValues(getMatInteriorCoarseByComponent(rowComponent),
1, &globalRowIndex, colsOther.size(),
&(colsOther[0]), &(valuesOther[0]), ADD_VALUES);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment