Commit 27235c55 authored by Thomas Witkowski's avatar Thomas Witkowski

Parallel type changes.

parent a8523376
......@@ -97,6 +97,10 @@ namespace AMDiS {
class VertexInfo;
class VertexVector;
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
class FeSpaceDofMap;
#endif
struct BoundaryObject;
struct AtomicBoundary;
......
......@@ -26,6 +26,9 @@
#include "BoundaryManager.h"
#include "Assembler.h"
#include "Serializer.h"
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
#include "parallel/ParallelDofMapping.h"
#endif
namespace AMDiS {
......@@ -233,11 +236,12 @@ namespace AMDiS {
if (condition && condition->isDirichlet()) {
if (condition->applyBoundaryCondition()) {
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
if ((*rankDofs)[rowIndices[i]])
applyDBCs.insert(static_cast<int>(row));
if (dofMap->isRankDof(rowIndices[i]))
applyDBCs.insert(static_cast<int>(row));
#else
applyDBCs.insert(static_cast<int>(row));
applyDBCs.insert(static_cast<int>(row));
#endif
}
} else {
......@@ -514,4 +518,10 @@ namespace AMDiS {
inserter = new inserter_type(matrix, nnz_per_row);
}
void DOFMatrix::setDofMap(FeSpaceDofMap& m)
{
dofMap = &m;
}
}
This diff is collapsed.
This diff is collapsed.
......@@ -946,8 +946,12 @@ namespace AMDiS {
template<typename T>
const bool DOFVector<T>::getDOFidxAtPoint(WorldVector<double> &p, DegreeOfFreedom &idx, ElInfo *oldElInfo, bool useOldElInfo) const
{ FUNCNAME("DOFVector::getDOFidxAtPoint()");
const bool DOFVector<T>::getDofIdxAtPoint(WorldVector<double> &p,
DegreeOfFreedom &idx,
ElInfo *oldElInfo,
bool useOldElInfo) const
{
FUNCNAME("DOFVector::getDofIdxAtPoint()");
Mesh *mesh = this->feSpace->getMesh();
const BasisFunction *basFcts = this->feSpace->getBasisFcts();
......@@ -994,7 +998,7 @@ namespace AMDiS {
if(!oldElInfo) delete elInfo;
return inside;
};
}
template<typename T>
......@@ -1052,7 +1056,7 @@ namespace AMDiS {
}
#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
this->rankDofs = rhs.rankDofs;
this->dofMap = rhs.dofMap;
#endif
return *this;
......
......@@ -695,7 +695,7 @@ namespace AMDiS {
TEST_EXIT(find(feSpaces.begin(), feSpaces.end(), rowFeSpace) != feSpaces.end())
("Should not happen!\n");
probStat->getSystemMatrix(i, j)->setRankDofs(dofFeData[rowFeSpace].isRankDof);
probStat->getSystemMatrix(i, j)->setDofMap(dofMap[rowFeSpace]);
}
......@@ -710,8 +710,8 @@ namespace AMDiS {
TEST_EXIT(find(feSpaces.begin(), feSpaces.end(), feSpace) != feSpaces.end())
("Should not happen!\n");
probStat->getRhsVector(i)->setRankDofs(dofFeData[feSpace].isRankDof);
probStat->getSolution(i)->setRankDofs(dofFeData[feSpace].isRankDof);
probStat->getRhsVector(i)->setDofMap(dofMap[feSpace]);
probStat->getSolution(i)->setDofMap(dofMap[feSpace]);
}
}
......@@ -2015,8 +2015,6 @@ namespace AMDiS {
DofContainer rankDofs(rankDofSet.begin(), rankDofSet.end());
sort(rankDofs.begin(), rankDofs.end(), cmpDofsByValue);
int nRankAllDofs = rankDofs.size();
// === Traverse interior boundaries and get all DOFs on them. ===
......@@ -2042,27 +2040,15 @@ namespace AMDiS {
for (; !it.endDofIter(); it.nextDof())
dofMap[feSpace].insert(it.getDofIndex());
// === Send and receive new DOF indices. ===
// === Update DOF admins due to new number of DOFs. ===
lastMeshChangeIndex = mesh->getChangeIndex();
#if (DEBUG != 0)
ParallelDebug::testDofContainerCommunication(*this,
sendDofs.getData(),
recvDofs.getData());
#endif
// First, we set all DOFs in ranks partition to be owend by the rank. Than,
// the DOFs in ranks partition that are owned by other rank are set to false.
dofFeData[feSpace].isRankDof.clear();
for (int i = 0; i < nRankAllDofs; i++)
dofFeData[feSpace].isRankDof[i] = true;
for (DofComm::Iterator it(recvDofs, feSpace); !it.end(); it.nextRank())
for (; !it.endDofIter(); it.nextDof())
dofFeData[feSpace].isRankDof[it.getDofIndex()] = false;
// === Update DOF admins due to new number of DOFs. ===
lastMeshChangeIndex = mesh->getChangeIndex();
}
......@@ -2278,15 +2264,11 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::mapGlobalToLocal()");
TEST_EXIT_DBG(dofFeData.count(feSpace))("Should not happen!\n");
ERROR_EXIT("REIMPLEMENT THAT SHIT!\n");
/*
for (DofMap::iterator it = dofFeData[feSpace].mapDofToGlobal.begin();
it != dofFeData[feSpace].mapDofToGlobal.end(); ++it)
if (it->second == dof)
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap[feSpace].getMap().begin();
it != dofMap[feSpace].getMap().end(); ++it)
if (it->second.global == dof)
return it->first;
*/
return -1;
}
......@@ -2313,18 +2295,7 @@ namespace AMDiS {
// === Serialieze FE space dependent data ===
unsigned int nFeSpace = feSpaces.size();
SerUtil::serialize(out, nFeSpace);
for (unsigned int i = 0; i < nFeSpace; i++) {
// SerUtil::serialize(out, dofFeData[feSpaces[i]].nRankDofs);
// SerUtil::serialize(out, dofFeData[feSpaces[i]].nOverallDofs);
// SerUtil::serialize(out, dofFeData[feSpaces[i]].rStartDofs);
SerUtil::serialize(out, dofFeData[feSpaces[i]].isRankDof);
// SerUtil::serialize(out, dofFeData[feSpaces[i]].mapDofToGlobal);
// SerUtil::serialize(out, dofFeData[feSpaces[i]].mapLocalToDof);
}
dofMap.serialize(out);
periodicMap.serialize(out, feSpaces);
......@@ -2385,21 +2356,7 @@ namespace AMDiS {
// === Deerialieze FE space dependent data ===
unsigned int nFeSpace = 0;
SerUtil::deserialize(in, nFeSpace);
TEST_EXIT(nFeSpace == feSpaces.size())
("Number of FE spaces is %d, but %d are stored in serialization file!\n",
feSpaces.size(), nFeSpace);
for (unsigned int i = 0; i < nFeSpace; i++) {
// SerUtil::deserialize(in, dofFeData[feSpaces[i]].nRankDofs);
// SerUtil::deserialize(in, dofFeData[feSpaces[i]].nOverallDofs);
// SerUtil::deserialize(in, dofFeData[feSpaces[i]].rStartDofs);
SerUtil::deserialize(in, dofFeData[feSpaces[i]].isRankDof);
// SerUtil::deserialize(in, dofFeData[feSpaces[i]].mapDofToGlobal);
// SerUtil::deserialize(in, dofFeData[feSpaces[i]].mapLocalToDof);
}
dofMap.deserialize(in);
periodicMap.deserialize(in, feSpaces);
......
......@@ -55,15 +55,6 @@ namespace AMDiS {
};
struct DofData
{
/// Maps all DOFs in ranks partition to a bool value. If it is true, the DOF
/// is owned by the rank. Otherwise, its an interior boundary DOF that is
/// owned by another rank.
DofIndexToBool isRankDof;
};
class MeshDistributor
{
private:
......@@ -222,11 +213,18 @@ namespace AMDiS {
/// Returns for a global index the DOF index in rank's subdomain. As there
/// is no direct data structure that stores this information, we have to
/// search for it in \ref dofFeData.mapDofToGlobal. This is not very
/// efficient and this function should thus be used for debugging only.
/// search for it in \ref dofMap. This is not very efficient and this
/// function should thus be used for debugging only.
DegreeOfFreedom mapGlobalToLocal(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof);
/// Maps a local DOF to its local index.
inline DegreeOfFreedom mapLocalToDof(const FiniteElemSpace *feSpace,
DegreeOfFreedom dof)
{
return dofMap[feSpace][dof].local;
}
/// Returns the periodic mapping handler, \ref periodicMap.
inline PeriodicMap& getPeriodicMap()
{
......@@ -252,15 +250,7 @@ namespace AMDiS {
/// is in rank's partition, but is owned by some other rank.
inline bool getIsRankDof(const FiniteElemSpace *feSpace, DegreeOfFreedom dof)
{
if (dofFeData[feSpace].isRankDof.count(dof))
return dofFeData[feSpace].isRankDof[dof];
return false;
}
inline DofIndexToBool& getIsRankDof(const FiniteElemSpace *feSpace)
{
return dofFeData[feSpace].isRankDof;
return dofMap[feSpace].isRankDof(dof);
}
inline long getLastMeshChangeIndex()
......@@ -565,8 +555,6 @@ namespace AMDiS {
/// macro element.
map<int, int> partitionMap;
map<const FiniteElemSpace*, DofData> dofFeData;
ParallelDofMapping dofMap;
/// Database to store and query all sub-objects of all elements of the
......
......@@ -198,7 +198,7 @@ namespace AMDiS {
for (PeriodicDofMap::iterator it = otherMap.begin();
it != otherMap.end(); ++it) {
for (DofMap::iterator dofIt = it->second.begin();
for (std::map<DegreeOfFreedom, DegreeOfFreedom>::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (dofMap.count(it->first) == 1 &&
dofMap[it->first].count(dofIt->first) == 1) {
......@@ -217,7 +217,7 @@ namespace AMDiS {
for (PeriodicDofMap::iterator it = dofMap.begin();
it != dofMap.end(); ++it) {
for (DofMap::iterator dofIt = it->second.begin();
for (std::map<DegreeOfFreedom, DegreeOfFreedom>::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (it->second[dofIt->second] != dofIt->first) {
MSG("[DBG] For boundary type %d: DOF %d -> %d, but %d -> %d!\n",
......@@ -668,42 +668,7 @@ namespace AMDiS {
{
FUNCNAME("ParallelDebug::printMapPeriodic()");
ERROR_EXIT("Function must be rewritten!\n");
#if 0
PeriodicMap &perMap = pdb.getPeriodicMap();
const FiniteElemSpace* feSpace = pdb.feSpaces[0];
typedef map<DegreeOfFreedom, DegreeOfFreedom> DofMap;
typedef map<DegreeOfFreedom, std::set<DegreeOfFreedom> > PeriodicDofMap;
if (rank == -1 || pdb.mpiRank == rank) {
cout << "====== DOF MAP PERIODIC ====== " << endl;
for (PeriodicDofMap::iterator it = perMap.periodicDofMap.begin();
it != perMap.periodicDofMap.end(); ++it) {
cout << "DOF MAP " << it->first << ": ";
for (std::set<DegreeOfFreedom>::iterator dofit = it->second.begin();
dofit != it->second.end(); ++dofit)
cout << *dofit << " ";
cout << endl;
DegreeOfFreedom localdof = -1;
map<DegreeOfFreedom, MultiIndex> &dofMap = pdb.dofMap[feSpace].getMap();
for (map<DegreeOfFreedom, MultiIndex>::iterator it = dofMap.begin();
it != dofMap.end(); ++it)
if (dofIt->second == it->first)
localdof = dofIt->first;
TEST_EXIT(localdof != -1)("There is something wrong!\n");
WorldVector<double> coords;
pdb.mesh->getDofIndexCoords(localdof, feSpace, coords);
coords.print();
}
}
#endif
ERROR_EXIT("Function must be rewritten, check svn for old code!\n");
}
......
......@@ -32,6 +32,7 @@
#define AMDIS_FE_SPACE_MAPPING_H
namespace AMDiS {
using namespace std;
/// Is used if a DOF index is mapped to multiple indices, i.e., to both, a local
......@@ -366,6 +367,18 @@ namespace AMDiS {
return dofToMatIndex.get(ithComponent, d) - rStartDofs;
}
// Writes all data of this object to an output stream.
void serialize(ostream &out)
{
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
// Reads the object data from an input stream.
void deserialize(istream &in)
{
ERROR_EXIT("MUST BE IMPLEMENTED!\n");
}
protected:
/// Insert a new FE space DOF mapping for a given FE space.
void addFeSpace(const FiniteElemSpace* feSpace);
......
......@@ -49,7 +49,7 @@ namespace AMDiS {
typedef map<int, DofContainer> RankToDofContainer;
/// Defines a mapping type from DOF indices to DOF indices.
typedef map<DegreeOfFreedom, DegreeOfFreedom> DofMap;
//typedef map<DegreeOfFreedom, DegreeOfFreedom> DofMap;
/// Defines a mapping type from DOFs to boolean values.
typedef map<const DegreeOfFreedom*, bool> DofToBool;
......@@ -61,6 +61,15 @@ namespace AMDiS {
typedef InteriorBoundary::RankToBoundMap RankToBoundMap;
typedef map<const DegreeOfFreedom*, DegreeOfFreedom> DofIndexMap;
/// Maps a boundary type, i.e., a boundary identifier index, to a periodic
/// DOF mapping.
typedef std::map<BoundaryType, map<DegreeOfFreedom, DegreeOfFreedom> > PeriodicDofMap;
/// Different FE spaces may have different DOFs on the same mesh. Thus we
/// need to have a periodic DOF mapping for each FE space.
typedef std::map<const FiniteElemSpace*, PeriodicDofMap> PeriodicDofMapFeSpace;
typedef vector<MeshStructure> MeshCodeVec;
}
......
......@@ -10,20 +10,26 @@
// See also license.opensource.txt in the distribution.
#include <map>
#include <fstream>
#include "Global.h"
#include "parallel/PeriodicMap.h"
#include "parallel/ParallelTypes.h"
namespace AMDiS {
using namespace std;
void PeriodicMap::add(const FiniteElemSpace *feSpace,
PeriodicDofMap &newMap)
{
FUNCNAME("PeriodicMap::add()");
for (PeriodicDofMap::iterator it = newMap.begin(); it != newMap.end(); ++it)
for (DofMap::iterator dofIt =it->second.begin();
for (PeriodicDofMap::iterator it = newMap.begin(); it != newMap.end(); ++it) {
for (std::map<DegreeOfFreedom, DegreeOfFreedom>::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt)
add(feSpace, it->first, dofIt->second, dofIt->first);
}
}
......@@ -64,7 +70,7 @@ namespace AMDiS {
for (PeriodicDofMap::iterator it = data.begin(); it != data.end(); ++it) {
int type = it->first;
DofMap dofMap = it->second;
std::map<DegreeOfFreedom, DegreeOfFreedom> dofMap = it->second;
SerUtil::serialize(out, type);
SerUtil::serialize(out, dofMap);
......@@ -97,7 +103,7 @@ namespace AMDiS {
for (int i = 0; i < mapSize; i++) {
int type;
DofMap dofMap;
std::map<DegreeOfFreedom, DegreeOfFreedom> dofMap;
SerUtil::deserialize(in, type);
SerUtil::deserialize(in, dofMap);
......
......@@ -32,15 +32,6 @@
namespace AMDiS {
/// Maps a boundary type, i.e., a boundary identifier index, to a periodic
/// DOF mapping.
typedef std::map<BoundaryType, DofMap> PeriodicDofMap;
/// Different FE spaces may have different DOFs on the same mesh. Thus we
/// need to have a periodic DOF mapping for each FE space.
typedef std::map<const FiniteElemSpace*, PeriodicDofMap> PeriodicDofMapFeSpace;
/** \brief
* This class stores information about the periodic DOFs in the (sub)domain.
* To each DOF on a periodic boundary there is the information to which DOF
......
......@@ -154,9 +154,8 @@ namespace AMDiS {
PetscScalar *vecPointer;
VecGetArray(tmp, &vecPointer);
ERROR_EXIT("REWRITE CODE!\n");
// for (int j = 0; j < nRankDofs; j++)
// dofvec[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[j];
for (int j = 0; j < nRankDofs; j++)
dofvec[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[j];
VecRestoreArray(tmp, &vecPointer);
}
......
......@@ -45,8 +45,7 @@ namespace AMDiS {
int recvAllValues = 0;
int sendValue =
static_cast<int>(meshDistributor->getLastMeshChangeIndex() != lastMeshNnz);
meshDistributor->getMpiComm().Allreduce(&sendValue, &recvAllValues,
1, MPI_INT, MPI_SUM);
mpiComm.Allreduce(&sendValue, &recvAllValues, 1, MPI_INT, MPI_SUM);
recvAllValues = 1;
......@@ -197,9 +196,8 @@ namespace AMDiS {
const FiniteElemSpace *feSpace = dv.getFeSpace();
int nRankDofs = meshDistributor->getNumberRankDofs(feSpace);
ERROR_EXIT("REWRITE CODE!\n");
// for (int j = 0; j < nRankDofs; j++)
// dv[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[c++];
for (int j = 0; j < nRankDofs; j++)
dv[meshDistributor->mapLocalToDof(feSpace, j)] = vecPointer[c++];
}
VecRestoreArray(petscSolVec, &vecPointer);
......
......@@ -100,6 +100,9 @@ namespace AMDiS {
otherBoundaryLocalDofs.insert(it.getDofIndex());
interiorDofs.clear();
ERROR_EXIT("Rewrite the following code block!\n");
#if 0
DofIndexToBool& isRankDof = meshDistributor->getIsRankDof(feSpace);
for (DofIndexToBool::iterator dofIt = isRankDof.begin();
dofIt != isRankDof.end(); ++dofIt) {
......@@ -108,6 +111,7 @@ namespace AMDiS {
otherBoundaryLocalDofs.count(dofIt->first) == 0)
interiorDofs.insert(meshDistributor->mapDofToGlobal(feSpace, dofIt->first));
}
#endif
nInteriorDofs = interiorDofs.size();
mpi::getDofNumbering(mpiComm, nInteriorDofs,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment