Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
Backofen, Rainer
amdis
Commits
cd501be4
Commit
cd501be4
authored
Jul 28, 2008
by
Thomas Witkowski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
* Small code changes in ParallelProblem
parent
50cb43cc
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
181 additions
and
106 deletions
+181
-106
AMDiS/src/ParMetisPartitioner.h
AMDiS/src/ParMetisPartitioner.h
+32
-18
AMDiS/src/ParallelProblem.cc
AMDiS/src/ParallelProblem.cc
+36
-82
AMDiS/src/ParallelProblem.h
AMDiS/src/ParallelProblem.h
+99
-0
AMDiS/src/PartitionElementData.h
AMDiS/src/PartitionElementData.h
+14
-6
No files found.
AMDiS/src/ParMetisPartitioner.h
View file @
cd501be4
...
...
@@ -50,25 +50,17 @@ namespace AMDiS {
~
ParMetisMesh
();
inline
void
setParMetisIndex
(
int
amdisIndex
,
int
parMetisIndex
)
{
// if(MPI::COMM_WORLD.Get_rank() == 0)
// MSG("set %p %d %d\n", this, amdisIndex, parMetisIndex);
inline
void
setParMetisIndex
(
int
amdisIndex
,
int
parMetisIndex
)
{
elem_a2p_
[
amdisIndex
]
=
parMetisIndex
+
1
;
};
inline
int
getParMetisIndex
(
int
amdisIndex
)
{
int
result
=
elem_a2p_
[
amdisIndex
];
// if(MPI::COMM_WORLD.Get_rank() == 0)
// MSG("get %p %d %d\n", this, amdisIndex, result - 1);
TEST_EXIT
(
result
>
0
)(
"invalid index
\n
"
);
return
result
-
1
;
};
inline
void
setAMDiSIndex
(
int
parMetisIndex
,
int
amdisIndex
)
{
inline
void
setAMDiSIndex
(
int
parMetisIndex
,
int
amdisIndex
)
{
elem_p2a_
[
parMetisIndex
]
=
amdisIndex
;
};
...
...
@@ -80,12 +72,29 @@ namespace AMDiS {
return
elem_p2a_
;
};
inline
int
*
getElementPtr
()
{
return
eptr_
;
};
inline
int
*
getElementInd
()
{
return
eind_
;
};
inline
int
*
getElementDist
()
{
return
elmdist_
;
};
inline
int
getDim
()
{
return
dim_
;
};
inline
float
*
getXYZ
()
{
return
xyz_
;
};
inline
int
getNumElements
()
{
return
numElements_
;
};
inline
int
*
getElementPtr
()
{
return
eptr_
;
};
inline
int
*
getElementInd
()
{
return
eind_
;
};
inline
int
*
getElementDist
()
{
return
elmdist_
;
};
inline
int
getDim
()
{
return
dim_
;
};
inline
float
*
getXYZ
()
{
return
xyz_
;
};
inline
int
getNumElements
()
{
return
numElements_
;
};
protected:
int
*
eptr_
;
...
...
@@ -110,8 +119,13 @@ namespace AMDiS {
~
ParMetisGraph
();
inline
int
*
getXAdj
()
{
return
xadj_
;
};
inline
int
*
getAdjncy
()
{
return
adjncy_
;
};
inline
int
*
getXAdj
()
{
return
xadj_
;
};
inline
int
*
getAdjncy
()
{
return
adjncy_
;
};
protected:
ParMetisMesh
*
parMetisMesh_
;
...
...
AMDiS/src/ParallelProblem.cc
View file @
cd501be4
...
...
@@ -451,9 +451,8 @@ namespace AMDiS {
exchangeMeshStructureCodes
(
structures
);
// merge codes
int
rank
;
for
(
rank
=
0
;
rank
<
mpiSize_
;
rank
++
)
{
if
(
rank
!=
mpiRank_
)
{
for
(
int
rank
=
0
;
rank
<
mpiSize_
;
rank
++
)
{
if
(
rank
!=
mpiRank_
)
{
structures
[
mpiRank_
].
merge
(
&
structures
[
rank
]);
}
}
...
...
@@ -511,7 +510,7 @@ namespace AMDiS {
const
FiniteElemSpace
*
feSpace
=
rankSolutions
[
0
]
->
getFESpace
();
TEST_EXIT
(
feSpace
->
getMesh
()
==
mesh_
)(
"invalid mesh
\n
"
);
int
i
,
dim
=
mesh_
->
getDim
();
int
dim
=
mesh_
->
getDim
();
const
BasisFunction
*
basFcts
=
feSpace
->
getBasisFcts
();
int
numFcts
=
basFcts
->
getNumber
();
DegreeOfFreedom
*
coarseDOFs
=
GET_MEMORY
(
DegreeOfFreedom
,
numFcts
);
...
...
@@ -519,11 +518,6 @@ namespace AMDiS {
DOFAdmin
*
admin
=
feSpace
->
getAdmin
();
int
partition
;
// std::vector<std::map<WorldVector<double>, DegreeOfFreedom> > sortedSendDOFs;
// std::vector<std::map<WorldVector<double>, DegreeOfFreedom> > sortedRecvDOFs;
// sortedSendDOFs.resize(mpiSize_);
// sortedRecvDOFs.resize(mpiSize_);
std
::
vector
<
std
::
vector
<
DegreeOfFreedom
>
>
sendOrder
;
std
::
vector
<
std
::
vector
<
DegreeOfFreedom
>
>
recvOrder
;
sendOrder
.
resize
(
mpiSize_
);
...
...
@@ -531,45 +525,36 @@ namespace AMDiS {
std
::
set
<
int
>::
iterator
setIt
,
setBegin
,
setEnd
;
//std::map<Element*, std::set<int> > elementPartitions;
elementPartitions_
.
clear
();
// if(mpiRank_ == 0) {
// bool wait = true;
// while(wait) {}
// }
int
elementPartition
=
-
1
;
Element
*
coarseElement
=
NULL
;
TraverseStack
stack
;
ElInfo
*
elInfo
=
stack
.
traverseFirst
(
mesh_
,
-
1
,
Mesh
::
CALL_EVERY_EL_PREORDER
);
while
(
elInfo
)
{
while
(
elInfo
)
{
Element
*
element
=
elInfo
->
getElement
();
PartitionElementData
*
partitionData
=
dynamic_cast
<
PartitionElementData
*>
(
element
->
getElementData
(
PARTITION_ED
));
if
(
partitionData
)
{
if
(
partitionData
->
getLevel
()
==
0
)
{
if
(
partitionData
)
{
if
(
partitionData
->
getLevel
()
==
0
)
{
elementPartition
=
partitionVec_
[
element
->
getIndex
()];
}
PartitionStatus
status
=
partitionData
->
getPartitionStatus
();
if
(
status
!=
OUT
)
{
if
(
partitionData
->
getLevel
()
==
localCoarseGridLevel_
)
{
if
(
status
!=
OUT
)
{
if
(
partitionData
->
getLevel
()
==
localCoarseGridLevel_
)
{
basFcts
->
getLocalIndices
(
element
,
admin
,
coarseDOFs
);
// collect other partitions element belongs to
for
(
i
=
0
;
i
<
dim
+
1
;
i
++
)
{
for
(
int
i
=
0
;
i
<
dim
+
1
;
i
++
)
{
setBegin
=
vertexPartitions_
[
coarseDOFs
[
i
]].
begin
();
setEnd
=
vertexPartitions_
[
coarseDOFs
[
i
]].
end
();
// setBegin = vertexPartitions_[element->getDOF(i, 0)].begin();
// setEnd = vertexPartitions_[element->getDOF(i, 0)].end();
for
(
setIt
=
setBegin
;
setIt
!=
setEnd
;
++
setIt
)
{
for
(
setIt
=
setBegin
;
setIt
!=
setEnd
;
++
setIt
)
{
elementPartitions_
[
element
].
insert
(
*
setIt
/* - 1*/
);
}
}
...
...
@@ -578,25 +563,24 @@ namespace AMDiS {
}
if
(
element
->
isLeaf
())
{
if
(
element
->
isLeaf
())
{
basFcts
->
getLocalIndices
(
element
,
admin
,
fineDOFs
);
for
(
i
=
0
;
i
<
numFcts
;
i
++
)
{
if
(
status
==
OVERLAP
)
{
for
(
int
i
=
0
;
i
<
numFcts
;
i
++
)
{
if
(
status
==
OVERLAP
)
{
// send dofs
sendOrder
[
elementPartition
].
push_back
(
fineDOFs
[
i
]);
}
if
(
status
==
IN
)
{
if
(
status
==
IN
)
{
// recv dofs
TEST_EXIT
(
elementPartition
==
mpiRank_
)(
"???
\n
"
);
setBegin
=
elementPartitions_
[
coarseElement
].
begin
();
setEnd
=
elementPartitions_
[
coarseElement
].
end
();
for
(
setIt
=
setBegin
;
setIt
!=
setEnd
;
++
setIt
)
{
if
(
*
setIt
!=
mpiRank_
)
{
for
(
setIt
=
setBegin
;
setIt
!=
setEnd
;
++
setIt
)
{
if
(
*
setIt
!=
mpiRank_
)
{
recvOrder
[
*
setIt
].
push_back
(
fineDOFs
[
i
]);
//recvOrder[*setIt].push_back(element->getDOF(i, 0));
}
}
}
...
...
@@ -661,12 +645,12 @@ namespace AMDiS {
WorldVector<double> worldCoords;
for(i = 0; i < numFcts; i++) {
elInfo2->coordToWorld(*(basFcts->getCoords(i)), &worldCoords);
if(status == OVERLAP) {
if
(status == OVERLAP) {
// send dofs
//sortedSendDOFs[elementPartition][worldCoords] = fineDOFs[i];
sendOrder[elementPartition].push_back(fineDOFs[i]);
}
if(status == IN) {
if
(status == IN) {
// recv dofs
TEST_EXIT(elementPartition == mpiRank_)("???\n");
setBegin = elementPartitions_[element1].begin();
...
...
@@ -694,10 +678,8 @@ namespace AMDiS {
std
::
map
<
int
,
int
>
sendBufferSize
;
std
::
map
<
int
,
int
>
recvBufferSize
;
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
//int sendSize = static_cast<int>(sortedSendDOFs[partition].size());
//int recvSize = static_cast<int>(sortedRecvDOFs[partition].size());
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
int
sendSize
=
static_cast
<
int
>
(
sendOrder
[
partition
].
size
());
int
recvSize
=
static_cast
<
int
>
(
recvOrder
[
partition
].
size
());
...
...
@@ -705,9 +687,7 @@ namespace AMDiS {
recvBufferSize
[
partition
]
=
recvSize
;
if
(
sendSize
>
0
)
{
sendBuffer
[
partition
]
=
GET_MEMORY
(
double
,
sendSize
);
//std::map<WorldVector<double>, DegreeOfFreedom>::iterator dofIt;
std
::
vector
<
DegreeOfFreedom
>::
iterator
dofIt
;
//dofIt = sortedSendDOFs[partition].begin();
dofIt
=
sendOrder
[
partition
].
begin
();
double
*
bufferIt
,
*
bufferBegin
,
*
bufferEnd
;
bufferBegin
=
sendBuffer
[
partition
];
...
...
@@ -716,7 +696,6 @@ namespace AMDiS {
bufferIt
<
bufferEnd
;
++
bufferIt
,
++
dofIt
)
{
//*bufferIt = (*solution)[dofIt->second];
*
bufferIt
=
(
*
solution
)[
*
dofIt
];
}
}
...
...
@@ -725,13 +704,6 @@ namespace AMDiS {
}
}
// for(partition = 0; partition < mpiSize_; partition++) {
// MSG("rank %d -> rank %d: %d",
// mpiRank_, partition, sendBufferSize[partition]);
// MSG("rank %d <- rank %d: %d",
// mpiRank_, partition, recvBufferSize[partition]);
// }
// non-blocking sends
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
...
...
@@ -745,8 +717,6 @@ namespace AMDiS {
}
}
//MPI::COMM_WORLD.Barrier();
// blocking recieves
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
...
...
@@ -760,29 +730,14 @@ namespace AMDiS {
}
}
// if(mpiRank_ == 1) {
// for(i = 0; i < recvBufferSize[0]; i++) {
// MSG("recv: %f", recvBuffer[0][i]);
// }
// }
// if(mpiRank_ == 0) {
// for(i = 0; i < sendBufferSize[1]; i++) {
// MSG("send: %f", sendBuffer[1][i]);
// }
// }
// wait for end of communication
MPI
::
COMM_WORLD
.
Barrier
();
// copy values into rank solutions
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
//std::map<WorldVector<double>, DegreeOfFreedom>::iterator dofIt;
//dofIt = sortedRecvDOFs[partition].begin();
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
std
::
vector
<
DegreeOfFreedom
>::
iterator
dofIt
=
recvOrder
[
partition
].
begin
();
for
(
i
=
0
;
i
<
recvBufferSize
[
partition
];
i
++
)
{
//(*(rankSolutions[partition]))[dofIt->second] =
for
(
i
=
0
;
i
<
recvBufferSize
[
partition
];
i
++
)
{
(
*
(
rankSolutions
[
partition
]))[
*
dofIt
]
=
recvBuffer
[
partition
][
i
];
++
dofIt
;
...
...
@@ -791,9 +746,9 @@ namespace AMDiS {
}
// free send and recv buffers
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
if
(
sendBufferSize
[
partition
]
>
0
)
for
(
partition
=
0
;
partition
<
mpiSize_
;
partition
++
)
{
if
(
partition
!=
mpiRank_
)
{
if
(
sendBufferSize
[
partition
]
>
0
)
FREE_MEMORY
(
sendBuffer
[
partition
],
double
,
sendBufferSize
[
partition
]);
...
...
@@ -1523,7 +1478,7 @@ namespace AMDiS {
bool
openOverlap
,
std
::
map
<
Element
*
,
int
>
&
overlapDistance
)
{
int
i
,
dim
=
mesh_
->
getDim
();
int
dim
=
mesh_
->
getDim
();
TraverseStack
stack
;
ElInfo
*
elInfo
;
...
...
@@ -1534,25 +1489,25 @@ namespace AMDiS {
// first: partition elements ...
int
index
,
partition
;
elInfo
=
stack
.
traverseFirst
(
mesh_
,
-
1
,
Mesh
::
CALL_EVERY_EL_PREORDER
);
while
(
elInfo
)
{
while
(
elInfo
)
{
Element
*
element
=
elInfo
->
getElement
();
PartitionElementData
*
partitionData
=
dynamic_cast
<
PartitionElementData
*>
(
element
->
getElementData
(
PARTITION_ED
));
if
(
partitionData
)
{
if
(
partitionData
->
getLevel
()
==
0
)
{
if
(
partitionData
)
{
if
(
partitionData
->
getLevel
()
==
0
)
{
index
=
element
->
getIndex
();
partition
=
partitionVec_
[
index
];
}
if
(
partitionData
->
getLevel
()
==
level
)
{
for
(
i
=
0
;
i
<
dim
+
1
;
i
++
)
{
vertexPartitions_
[
element
->
getDOF
(
i
,
0
)].
insert
(
partition
/* + 1*/
);
if
(
partitionData
->
getLevel
()
==
level
)
{
for
(
int
i
=
0
;
i
<
dim
+
1
;
i
++
)
{
vertexPartitions_
[
element
->
getDOF
(
i
,
0
)].
insert
(
partition
);
}
}
}
elInfo
=
stack
.
traverseNext
(
elInfo
);
}
if
(
overlap
>
1
||
openOverlap
==
false
)
{
if
(
overlap
>
1
||
openOverlap
==
false
)
{
// exchange mesh structure codes
MeshStructure
*
structures
=
NEW
MeshStructure
[
mpiSize_
];
exchangeMeshStructureCodes
(
structures
);
...
...
@@ -2050,9 +2005,8 @@ namespace AMDiS {
std
::
vector
<
DOFVector
<
double
>*>
rankSol
(
mpiSize_
);
int
i
,
j
;
for
(
i
=
0
;
i
<
numComponents_
;
i
++
)
{
for
(
j
=
0
;
j
<
mpiSize_
;
j
++
)
{
for
(
int
i
=
0
;
i
<
numComponents_
;
i
++
)
{
for
(
int
j
=
0
;
j
<
mpiSize_
;
j
++
)
{
rankSol
[
j
]
=
rankSolution_
[
j
]
->
getDOFVector
(
i
);
}
...
...
AMDiS/src/ParallelProblem.h
View file @
cd501be4
...
...
@@ -262,31 +262,130 @@ namespace AMDiS {
double
errors2map
(
std
::
map
<
int
,
double
>
&
errMap
,
int
comp
,
bool
add
);
protected:
/** \brief
*
*/
std
::
string
name_
;
/** \brief
*
*/
Mesh
*
mesh_
;
/** \brief
*
*/
RefinementManager
*
refinementManager_
;
/** \brief
*
*/
CoarseningManager
*
coarseningManager_
;
/** \brief
* Pointer to the paritioner which is used to devide a mesh into partitions.
*/
ParMetisPartitioner
*
partitioner_
;
/** \brief
* Stores to every element index the number of the partition it corresponds to.
*/
std
::
map
<
int
,
int
>
partitionVec_
;
/** \brief
* Stores an old partitioning of elements. To every element index the number of
* the parition it corresponds to is stored.
*/
std
::
map
<
int
,
int
>
oldPartitionVec_
;
/** \brief
*
*/
std
::
map
<
int
,
double
>
elemWeights_
;
/** \brief
*
*/
std
::
map
<
Element
*
,
std
::
set
<
int
>
>
elementPartitions_
;
/** \brief
* Stores to every DOF the set of partitions it corresponds to.
*/
std
::
map
<
DegreeOfFreedom
,
std
::
set
<
int
>
>
vertexPartitions_
;
/** \brief
*
*/
int
repartitionSteps_
;
/** \brief
*
*/
bool
puEveryTimestep_
;
/** \brief
*
*/
std
::
vector
<
DOFVector
<
double
>*>
dofVectors_
;
/** \brief
*
*/
double
upperPartThreshold_
;
/** \brief
*
*/
double
lowerPartThreshold_
;
/** \brief
*
*/
int
globalCoarseGridLevel_
;
/** \brief
*
*/
int
localCoarseGridLevel_
;
/** \brief
*
*/
int
globalRefinements_
;
/** \brief
*
*/
std
::
map
<
Element
*
,
int
>
overlapDistance_
;
/** \brief
*
*/
int
adaptiveThresholds_
;
/** \brief
*
*/
double
thresholdIncFactor_
;
/** \brief
*
*/
double
thresholdDecFactor_
;
/** \brief
*
*/
double
repartTimeFactor_
;
/** \brief
*
*/
double
minUpperTH_
;
/** \brief
*
*/
double
maxLowerTH_
;
};
...
...
AMDiS/src/PartitionElementData.h
View file @
cd501be4
...
...
@@ -43,7 +43,7 @@ namespace AMDiS {
MEMORY_MANAGED
(
PartitionElementData
);
inline
bool
isOfType
(
int
typeID
)
const
{
if
(
typeID
==
PARTITION_ED
)
if
(
typeID
==
PARTITION_ED
)
return
true
;
return
false
;
};
...
...
@@ -85,9 +85,13 @@ namespace AMDiS {
return
newObj
;
};
inline
std
::
string
getTypeName
()
const
{
return
"PartitionElementData"
;
};
inline
std
::
string
getTypeName
()
const
{
return
"PartitionElementData"
;
};
inline
const
int
getTypeID
()
const
{
return
PARTITION_ED
;
};
inline
const
int
getTypeID
()
const
{
return
PARTITION_ED
;
};
void
serialize
(
std
::
ostream
&
out
)
{
...
...
@@ -109,17 +113,21 @@ namespace AMDiS {
status_
=
status
;
};
inline
PartitionStatus
getPartitionStatus
()
{
return
status_
;
};
inline
PartitionStatus
getPartitionStatus
()
{
return
status_
;
};
inline
void
setLevel
(
int
level
)
{
level_
=
level
;
};
inline
int
getLevel
()
{
return
level_
;
};
inline
int
getLevel
()
{
return
level_
;
};
void
descend
(
Element
*
element
)
{
if
(
!
element
->
isLeaf
())
{
if
(
!
element
->
isLeaf
())
{
Element
*
child0
=
element
->
getChild
(
0
);
Element
*
child1
=
element
->
getChild
(
1
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment