Skip to content

GitLab

Commit dfeb9e9e authored by Thomas Witkowski's avatar Thomas Witkowski
Browse files
Options

Work on parallel periodic boundary condition in 3D.

parent 32b71148
Hide whitespace changes
Inline Side-by-side
Showing with 289 additions and 67 deletions
AMDiS/src/parallel/MeshDistributor.cc
View file @ dfeb9e9e
......@@ -221,6 +221,9 @@ namespace AMDiS {
createPeriodicMap();
#if (DEBUG != 0)
ParallelDebug::testPeriodicBoundary(*this);
#endif
// === Global refinements. ===
......@@ -547,7 +550,7 @@ namespace AMDiS {
for (InteriorBoundary::iterator it(periodicBoundary); !it.end(); ++it) {
if (it.getRank() == mpiRank) {
// ERROR_EXIT("Na, du weisst schon!\n");
WARNING("Na, du weisst schon!\n");
} else {
if ((mesh->getDim() == 2 && it->rankObj.subObj == EDGE) ||
(mesh->getDim() == 3 && it->rankObj.subObj == FACE))
......@@ -1261,8 +1264,6 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::createInteriorBoundaryInfo()");
MSG("CREATE BOUNDARY INFO!\n");
createMeshElementData();
createBoundaryData();
......@@ -1274,8 +1275,6 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::updateInteriorBoundaryInfo()");
MSG("UPDATE BOUNDARY INFO!\n");
elObjects.createRankData(partitionVec);
createBoundaryData();
......@@ -1328,8 +1327,8 @@ namespace AMDiS {
periodicEdgeAssoc[edge1].insert(edge2);
periodicDofs[std::make_pair(edge1.first, edge2.first)] = boundaryType;
periodicDofs[std::make_pair(edge1.second, edge2.second)] = boundaryType;
periodicVertices[std::make_pair(edge1.first, edge2.first)] = boundaryType;
periodicVertices[std::make_pair(edge1.second, edge2.second)] = boundaryType;
periodicDofAssoc[edge1.first].insert(boundaryType);
periodicDofAssoc[edge1.second].insert(boundaryType);
......@@ -1351,9 +1350,9 @@ namespace AMDiS {
periodicFaces[std::make_pair(face1, face2)] = elInfo->getBoundary(i);
periodicDofs[std::make_pair(face1.get<0>(), face2.get<0>())] = boundaryType;
periodicDofs[std::make_pair(face1.get<1>(), face2.get<1>())] = boundaryType;
periodicDofs[std::make_pair(face1.get<2>(), face2.get<2>())] = boundaryType;
periodicVertices[std::make_pair(face1.get<0>(), face2.get<0>())] = boundaryType;
periodicVertices[std::make_pair(face1.get<1>(), face2.get<1>())] = boundaryType;
periodicVertices[std::make_pair(face1.get<2>(), face2.get<2>())] = boundaryType;
periodicDofAssoc[face1.get<0>()].insert(boundaryType);
periodicDofAssoc[face1.get<1>()].insert(boundaryType);
......@@ -1395,7 +1394,7 @@ namespace AMDiS {
// === Search for interectly connected vertices in periodic boundaries. ===
if (periodicDofs.size() > 0) {
if (periodicVertices.size() > 0) {
// === Search for an unsed boundary index. ===
......@@ -1413,52 +1412,51 @@ namespace AMDiS {
// === Get all vertex DOFs that have multiple periodic associations. ===
std::vector<DegreeOfFreedom> multiplePeriodicDof;
std::map<int, std::vector<DegreeOfFreedom> > multiplePeriodicDof;
for (std::map<DegreeOfFreedom, std::set<BoundaryType> >::iterator it = periodicDofAssoc.begin();
it != periodicDofAssoc.end(); ++it) {
if (mesh->getDim() == 2) {
TEST_EXIT_DBG(it->second.size() <= 2)("Should not happen!\n");
}
if (mesh->getDim() == 3) {
TEST_EXIT_DBG(it->second.size() <= 3)("Should not happen!\n");
}
if ((mesh->getDim() == 2 && it->second.size() == 2) ||
(mesh->getDim() == 3 && it->second.size() == 3))
multiplePeriodicDof.push_back(it->first);
}
if (mesh->getDim() == 2) {
TEST_EXIT_DBG(multiplePeriodicDof.size() == 0 ||
multiplePeriodicDof.size() == 4)
("Should not happen (%d)!\n", multiplePeriodicDof.size());
}
if (mesh->getDim() == 3) {
TEST_EXIT_DBG(multiplePeriodicDof.size() == 0 ||
multiplePeriodicDof.size() == 8)
("Should not happen (%d)!\n", multiplePeriodicDof.size());
TEST_EXIT_DBG((mesh->getDim() == 2 && it->second.size() <= 2) ||
(mesh->getDim() == 3 && it->second.size() <= 3))
("Should not happen!\n");
multiplePeriodicDof[it->second.size()].push_back(it->first);
}
int nMultiplePeriodicDofs = multiplePeriodicDof.size();
for (int i = 0; i < nMultiplePeriodicDofs; i++) {
for (int j = 0; j < nMultiplePeriodicDofs; j++) {
if (i == j)
continue;
std::pair<DegreeOfFreedom, DegreeOfFreedom> perDofs =
std::make_pair(multiplePeriodicDof[i], multiplePeriodicDof[j]);
if (periodicDofs.count(perDofs) == 0)
periodicDofs[perDofs] = newPeriodicBoundaryType;
if (mesh->getDim() == 2)
TEST_EXIT_DBG(multiplePeriodicDof[2].size() == 0 ||
multiplePeriodicDof[2].size() == 4)
("Should not happen (%d)!\n", multiplePeriodicDof[2].size());
if (mesh->getDim() == 3)
TEST_EXIT_DBG(multiplePeriodicDof[3].size() == 0 ||
multiplePeriodicDof[3].size() == 8)
("Should not happen (%d)!\n", multiplePeriodicDof[3].size());
for (int k = 2; k <= 3; k++) {
int nMultiplePeriodicDofs = multiplePeriodicDof[k].size();
for (int i = 0; i < nMultiplePeriodicDofs; i++) {
for (int j = i + 1; j < nMultiplePeriodicDofs; j++) {
std::pair<DegreeOfFreedom, DegreeOfFreedom> perDofs0 =
std::make_pair(multiplePeriodicDof[k][i], multiplePeriodicDof[k][j]);
std::pair<DegreeOfFreedom, DegreeOfFreedom> perDofs1 =
std::make_pair(multiplePeriodicDof[k][j], multiplePeriodicDof[k][i]);
if (periodicVertices.count(perDofs0) == 0) {
TEST_EXIT_DBG(periodicVertices.count(perDofs1) == 0)
("Should not happen!\n");
periodicVertices[perDofs0] = newPeriodicBoundaryType;
periodicVertices[perDofs1] = newPeriodicBoundaryType;
newPeriodicBoundaryType--;
mesh->getPeriodicAssociations()[newPeriodicBoundaryType] =
new VertexVector(feSpace->getAdmin(), "");
}
}
}
}
// === Get all edges that have multiple periodic associations (3D only!). ===
for (std::map<DofEdge, std::set<DofEdge> >::iterator it = periodicEdgeAssoc.begin();
......@@ -1474,10 +1472,47 @@ namespace AMDiS {
periodicEdges[perEdge0] = newPeriodicBoundaryType;
periodicEdges[perEdge1] = newPeriodicBoundaryType;
newPeriodicBoundaryType--;
mesh->getPeriodicAssociations()[newPeriodicBoundaryType] =
new VertexVector(feSpace->getAdmin(), "");
}
}
// === In debug mode we make some tests, if the periodic structures are set ===
// === in a symmetric way, i.e., if A -> B for a specific boundary type, ===
// === there must be a mapping B -> A with the same boundary type. ===
#if (DEBUG != 0)
for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicVertices.begin();
it != periodicVertices.end(); ++it) {
std::pair<DegreeOfFreedom, DegreeOfFreedom> testVertex =
std::make_pair(it->first.second, it->first.first);
TEST_EXIT_DBG(periodicVertices.count(testVertex) == 1)("Should not happen!\n");
TEST_EXIT_DBG(periodicVertices[testVertex] == it->second)("Should not happen!\n");
}
for (std::map<std::pair<DofEdge, DofEdge>, BoundaryType>::iterator it = periodicEdges.begin();
it != periodicEdges.end(); ++it) {
std::pair<DofEdge, DofEdge> testEdge =
std::make_pair(it->first.second, it->first.first);
TEST_EXIT_DBG(periodicEdges.count(testEdge) == 1)("Should not happen!\n");
TEST_EXIT_DBG(periodicEdges[testEdge] == it->second)("Should not happen!\n");
}
for (std::map<std::pair<DofFace, DofFace>, BoundaryType>::iterator it = periodicFaces.begin();
it != periodicFaces.end(); ++it) {
std::pair<DofFace, DofFace> testFace =
std::make_pair(it->first.second, it->first.first);
TEST_EXIT_DBG(periodicFaces.count(testFace) == 1)("Should not happen!\n");
TEST_EXIT_DBG(periodicFaces[testFace] == it->second)("Should not happen!\n");
}
#endif
}
}
......@@ -1488,7 +1523,7 @@ namespace AMDiS {
FUNCNAME("MeshDistributor::createBoundaryData()");
// === Clear all relevant data structures, ===
// === Clear all relevant data structures. ===
myIntBoundary.clear();
otherIntBoundary.clear();
......@@ -1578,12 +1613,19 @@ namespace AMDiS {
// === Create periodic boundary data structure. ===
for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicDofs.begin();
it != periodicDofs.end(); ++it) {
for (std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType>::iterator it = periodicVertices.begin();
it != periodicVertices.end(); ++it) {
if (elObjects.isInRank(it->first.first, mpiRank) == false)
continue;
WorldVector<double> c0, c1;
mesh->getDofIndexCoords(it->first.first, feSpace, c0);
mesh->getDofIndexCoords(it->first.second, feSpace, c1);
MSG("CREATE BOUNDARY FOR DOF MAP: %d (%.3f %.3f %.3f)<-> %d (%.3f %.3f %.3f)\n", it->first.first,
c0[0], c0[1], c0[2], it->first.second, c1[0], c1[1], c1[2]);
ElementObjectData& perDofEl0 = elObjects.getElementsInRank(it->first.first)[mpiRank];
MSG("DATA: %d %d %d\n", perDofEl0.elIndex, VERTEX, perDofEl0.ithObject);
for (std::map<int, ElementObjectData>::iterator elIt = elObjects.getElementsInRank(it->first.second).begin();
elIt != elObjects.getElementsInRank(it->first.second).end(); ++elIt) {
......@@ -2033,6 +2075,8 @@ namespace AMDiS {
for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin();
it != periodicBoundary.boundary.end(); ++it) {
MSG("------------- WITH RANK %d ------------------\n", it->first);
if (it->first == mpiRank) {
TEST_EXIT_DBG(it->second.size() % 2 == 0)("Should not happen!\n");
......@@ -2044,6 +2088,10 @@ namespace AMDiS {
bound.neighObj.el->getVertexDofs(feSpace, bound.neighObj, dofs1);
bound.neighObj.el->getNonVertexDofs(feSpace, bound.neighObj, dofs1);
MSG("BOUND-I %d-%d-%d WITH %d-%d-%d\n",
bound.rankObj.elIndex, bound.rankObj.subObj, bound.rankObj.ithObj,
bound.neighObj.elIndex, bound.neighObj.subObj, bound.neighObj.ithObj);
TEST_EXIT_DBG(dofs0.size() == dofs1.size())("Should not happen!\n");
BoundaryType type = bound.type;
......@@ -2055,6 +2103,8 @@ namespace AMDiS {
if (periodicDofAssociations[globalDof0].count(type) == 0) {
periodicDof[type][globalDof0] = globalDof1;
periodicDofAssociations[globalDof0].insert(type);
MSG("SET(A TYPE %d) DOF %d -> %d\n", type, globalDof0, globalDof1);
}
}
}
......@@ -2066,12 +2116,20 @@ namespace AMDiS {
boundIt != it->second.end(); ++boundIt) {
int nDofs = dofs.size();
MSG("BOUND-R (T = %d) %d-%d-%d WITH %d-%d-%d\n",
boundIt->type,
boundIt->rankObj.elIndex, boundIt->rankObj.subObj, boundIt->rankObj.ithObj,
boundIt->neighObj.elIndex, boundIt->neighObj.subObj, boundIt->neighObj.ithObj);
boundIt->rankObj.el->getVertexDofs(feSpace, boundIt->rankObj, dofs);
boundIt->rankObj.el->getNonVertexDofs(feSpace, boundIt->rankObj, dofs);
for (unsigned int i = 0; i < (dofs.size() - nDofs); i++)
for (unsigned int i = 0; i < (dofs.size() - nDofs); i++) {
MSG(" i = %d DOF = %d\n", nDofs + i, mapLocalGlobalDofs[*(dofs[nDofs + i])]);
rankToDofType[it->first].push_back(boundIt->type);
}
}
// Send the global indices to the rank on the other side.
......@@ -2088,6 +2146,8 @@ namespace AMDiS {
stdMpi.startCommunication<int>(MPI_INT);
MSG("===============================\n");
// === The rank has received the dofs from the rank on the other side of ===
// === the boundary. Now it can use them to create the mapping between ===
// === the periodic dofs in this rank and the corresponding periodic ===
......@@ -2096,6 +2156,9 @@ namespace AMDiS {
for (RankToBoundMap::iterator it = periodicBoundary.boundary.begin();
it != periodicBoundary.boundary.end(); ++it) {
MSG("------------- WITH RANK %d ------------------\n", it->first);
DofContainer& dofs = rankPeriodicDofs[it->first];
std::vector<int>& types = rankToDofType[it->first];
......@@ -2110,24 +2173,29 @@ namespace AMDiS {
// Check if this global dof with the corresponding boundary type was
// not added before by another periodic boundary from other rank.
if (periodicDofAssociations[globalDofIndex].count(type) == 0) {
MSG("SET(B-%d TYPE %d) DOF %d -> %d\n", i, type, globalDofIndex, mapGlobalDofIndex);
periodicDof[type][globalDofIndex] = mapGlobalDofIndex;
periodicDofAssociations[globalDofIndex].insert(type);
} else {
MSG("ASSOC ALREADY SET FOR %d TYPE %d\n", i, type);
}
}
}
#if (DEBUG != 0)
// In 2D, a periodic DOF can have either 1 or 3 associations. Check this!
if (mesh->getDim() == 2) {
for (std::map<int, std::set<BoundaryType> >::iterator it = periodicDofAssociations.begin();
it != periodicDofAssociations.end(); ++it) {
int nAssoc = it->second.size();
TEST_EXIT_DBG(nAssoc == 1 || nAssoc == 3)
("Should not happen! DOF %d has %d periodic associations!\n",
it->first, nAssoc);
}
}
for (std::map<int, std::set<BoundaryType> >::iterator it = periodicDofAssociations.begin();
it != periodicDofAssociations.end(); ++it) {
WorldVector<double> c;
mesh->getDofIndexCoords(it->first, feSpace, c);
int nAssoc = it->second.size();
TEST_EXIT_DBG(nAssoc == 1 || nAssoc == 3 ||
(mesh->getDim() == 3 && (nAssoc == 7 || nAssoc == 11)))
("Should not happen! DOF %d (%e %e %e) has %d periodic associations!\n",
it->first, c[0], c[1], (mesh->getDim() == 2 ? 0.0 : c[2]), nAssoc);
}
#endif
}
......@@ -2179,7 +2247,7 @@ namespace AMDiS {
elObjects.serialize(out);
SerUtil::serialize(out, periodicDofs);
SerUtil::serialize(out, periodicVertices);
SerUtil::serialize(out, periodicEdges);
SerUtil::serialize(out, periodicFaces);
......@@ -2243,7 +2311,7 @@ namespace AMDiS {
elObjects.deserialize(in);
SerUtil::deserialize(in, periodicDofs);
SerUtil::deserialize(in, periodicVertices);
SerUtil::deserialize(in, periodicEdges);
SerUtil::deserialize(in, periodicFaces);
......
AMDiS/src/parallel/MeshDistributor.h
View file @ dfeb9e9e
......@@ -479,7 +479,7 @@ namespace AMDiS {
std::map<int, int> macroElIndexTypeMap;
// The following three data structures store periodic DOFs, edges and faces.
std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicDofs;
std::map<std::pair<DegreeOfFreedom, DegreeOfFreedom>, BoundaryType> periodicVertices;
std::map<std::pair<DofEdge, DofEdge>, BoundaryType> periodicEdges;
std::map<std::pair<DofFace, DofFace>, BoundaryType> periodicFaces;
......
AMDiS/src/parallel/ParallelDebug.cc
View file @ dfeb9e9e
......@@ -139,6 +139,89 @@ namespace AMDiS {
}
void ParallelDebug::testPeriodicBoundary(MeshDistributor &pdb)
{
FUNCNAME("ParallelDebug::testPeriodicBoundary()");
typedef MeshDistributor::PeriodicDofMap PeriodicDofMap;
StdMpi<PeriodicDofMap> stdMpi(pdb.mpiComm, true);
if (pdb.mpiRank == 0) {
for (int i = 1; i < pdb.mpiSize; i++)
stdMpi.recv(i);
} else {
stdMpi.send(0, pdb.periodicDof);
}
stdMpi.startCommunication<int>(MPI_INT);
int foundError = 0;
// === The boundary DOFs are checked only on the zero rank. ===
if (pdb.mpiRank == 0) {
std::map<int, PeriodicDofMap> rankToMaps;
PeriodicDofMap dofMap = pdb.periodicDof;
rankToMaps[0] = dofMap;
for (int i = 1; i < pdb.mpiSize; i++) {
PeriodicDofMap &otherMap = stdMpi.getRecvData(i);
rankToMaps[i] = otherMap;
for (PeriodicDofMap::iterator it = otherMap.begin();
it != otherMap.end(); ++it) {
for (MeshDistributor::DofMapping::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (dofMap.count(it->first) == 1 &&
dofMap[it->first].count(dofIt->first) == 1) {
TEST_EXIT_DBG(dofMap[it->first][dofIt->first] == dofIt->second)
("Should not happen!\n");
} else {
dofMap[it->first][dofIt->first] = dofIt->second;
}
}
}
}
// === Now we test if global DOF A is mapped to B, then B must be mapped ===
// === to A for the same boundary type. ===
for (PeriodicDofMap::iterator it = dofMap.begin();
it != dofMap.end(); ++it) {
for (MeshDistributor::DofMapping::iterator dofIt = it->second.begin();
dofIt != it->second.end(); ++dofIt) {
if (it->second[dofIt->second] != dofIt->first) {
MSG("[DBG] For boundary type %d: DOF %d -> %d, but %d -> %d!\n",
it ->first,
dofIt->first, dofIt->second,
dofIt->second, it->second[dofIt->second]);
for (int i = 0; i < pdb.mpiSize; i++) {
if (rankToMaps[i][it->first].count(dofIt->first) == 1) {
MSG("[DBG] %d -> %d in rank %d\n",
dofIt->first, rankToMaps[i][it->first][dofIt->first], i);
}
if (rankToMaps[i][it->first].count(dofIt->second) == 1) {
MSG("[DBG] %d -> %d in rank %d\n",
dofIt->second, rankToMaps[i][it->first][dofIt->second], i);
}
}
ERROR("Wrong periodic DOFs!\n");
foundError = 1;
}
}
}
}
mpi::globalAdd(foundError);
TEST_EXIT(foundError == 0)("Error found on at least on rank!\n");
}
void ParallelDebug::testCommonDofs(MeshDistributor &pdb, bool printCoords)
{
FUNCNAME("ParallelDebug::testCommonDofs()");
......@@ -573,7 +656,8 @@ namespace AMDiS {
}
for (InteriorBoundary::iterator it(pdb.periodicBoundary); !it.end(); ++it) {
MSG("Periodic boundary with rank %d: \n", it.getRank());
MSG("Periodic boundary (ID %d) with rank %d: \n",
it->type, it.getRank());
MSG(" ranks obj-ind: %d sub-obj: %d ith-obj: %d\n",
it->rankObj.elIndex, it->rankObj.subObj, it->rankObj.ithObj);
MSG(" neigh obj-ind: %d sub-obj: %d ith-obj: %d\n",
......
AMDiS/src/parallel/ParallelDebug.h
View file @ dfeb9e9e
......@@ -41,6 +41,13 @@ namespace AMDiS {
*/
static void testInteriorBoundary(MeshDistributor &pdb);
/** \brief
* Test if all periodic boundaries are set in a consistent way on all ranks.
*
* \param[in] pdb Parallel problem definition used for debugging.
*/
static void testPeriodicBoundary(MeshDistributor &pdb);
/** \brief
* This function is used for debugging only. It traverses all interior boundaries
* and compares the DOF indices on them with the dof indices of the boundarys
......
AMDiS/src/parallel/StdMpi.cc
View file @ dfeb9e9e
......@@ -83,6 +83,20 @@ namespace AMDiS {
return size;
}
int intSizeOf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data)
{
int size = 1;
for (std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> >::iterator it = data.begin();
it != data.end(); ++it) {
size += 2 + it->second.size() * 2;
}
MSG("RET SIZE = %d\n", size);
return size;
}
void makeBuf(int &data, int *buf)
{
buf[0] = data;
......@@ -322,4 +336,48 @@ namespace AMDiS {
}
}
void makeBuf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf)
{
buf[0] = data.size();
int counter = 1;
for (std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> >::iterator it = data.begin();
it != data.end(); ++it) {
buf[counter++] = it->first;
buf[counter++] = it->second.size();
for (std::map<DegreeOfFreedom, DegreeOfFreedom>::iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2) {
buf[counter++] = it2->first;
buf[counter++] = it2->second;
}
}
}
void makeFromBuf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf, int bufSize)
{
data.clear();
if (bufSize == 0)
return;
int counter = 1;
for (int i = 0; i < buf[0]; i++) {
BoundaryType bound = buf[counter++];
std::map<DegreeOfFreedom, DegreeOfFreedom> dofs;
int nDofs = buf[counter++];
for (int j = 0; j < nDofs; j++) {
DegreeOfFreedom dof0, dof1;
dof0 = buf[counter++];
dof1 = buf[counter++];
dofs[dof0] = dof1;
}
data[bound] = dofs;
}
TEST_EXIT(bufSize == counter)("Should not happen!\n");
}
}
AMDiS/src/parallel/StdMpi.h
View file @ dfeb9e9e
......@@ -55,6 +55,8 @@ namespace AMDiS {
int intSizeOf(std::vector<std::vector<double> > &data);
int intSizeOf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data);
void makeBuf(int &data, int *buf);
void makeFromBuf(int &data, int *buf, int bufSize);
......@@ -98,7 +100,10 @@ namespace AMDiS {
void makeBuf(std::vector<std::vector<double> > &data, double *buf);
void makeFromBuf(std::vector<std::vector<double> > &data, double *buf, int bufSize);
void makeBuf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf);
void makeFromBuf(std::map<BoundaryType, std::map<DegreeOfFreedom, DegreeOfFreedom> > &data, int *buf, int bufSize);
template<typename SendT, typename RecvT=SendT>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment