diff --git a/AMDiS/AMDISConfig.cmake.in b/AMDiS/AMDISConfig.cmake.in
index b7cd1b36b5adb780e74fd434d5a8e6a4d73e0c56..34e34e6ed2f5757e5eb0ff54d025715e62145576 100644
--- a/AMDiS/AMDISConfig.cmake.in
+++ b/AMDiS/AMDISConfig.cmake.in
@@ -69,63 +69,36 @@ set(AMDIS_MAJOR_VERSION @AMDIS_MAJOR@)
set(AMDIS_MINOR_VERSION @AMDIS_MINOR@)
if(AMDIS_NEED_UMFPACK)
- #look for umfpack, we need the headers only
- find_file(UMFPACK_H umfpack.h
- HINTS /usr/include /usr/include/suitesparse /usr/include/ufsparse
- DOC "headerfile umfpack.h for UMFPACK")
- if( UMFPACK_H )
- #construct the include path
- get_filename_component(UMFPACK_PATH ${UMFPACK_H} PATH)
- #look for ufconfig
- find_file(UFCONFIG_H UFconfig.h
- HINTS /usr/include ${UMFPACK_PATH}
- DOC "headerfile UFConfig.h")
- if( UFCONFIG_H )
- get_filename_component(UFCONFIG_PATH ${UFCONFIG_H} PATH)
- else()
- message(FATAL_ERROR "could not find UFConfig.h, needed for umfpack")
- endif()
- #look for amd
- find_file(AMD_H amd.h
- HINTS /usr/include/ ${UMFPACK_PATH}
- DOC "headerfile amd.h")
- if( AMD_H )
- get_filename_component(AMD_PATH ${AMD_H} PATH)
- else()
- message( FATAL_ERROR "could not find amd.h, needed for umfpack")
- endif()
-
- #add the compileflags and directories
- list(APPEND AMDIS_INCLUDE_DIRS ${UMFPACK_PATH} ${UFCONFIG_PATH} ${AMD_PATH})
- else()
- message(FATAL_ERROR "Could not find the umfpack-headers. Please install umfpack")
- endif()
- find_library(BLAS_LIBRARY NAMES "blas" "goto"
- DOC "the blas library")
-# message("blaslibrary: ${BLAS_LIBRARY}")
- if( BLAS_LIBRARY )
- list(APPEND AMDIS_LIBRARIES ${BLAS_LIBRARY})
- else()
- message(ERROR " could not find the blas library. please set the variable BLAS_LIBRARY to the blas library with full path")
- endif()
+ set(AMDIS_UMFPACK_PATH @UMFPACK_PATH@)
+ list(APPEND AMDIS_INCLUDE_DIRS ${AMDIS_UMFPACK_PATH})
find_library(UMFPACK_LIB umfpack
- DOC "the UMFPACK library")
- if( UMFPACK_LIB )
+ HINTS ENV LIBRARY_PATH
+ DOC "The UMFPACK library")
+ if(UMFPACK_LIB)
list(APPEND AMDIS_LIBRARIES ${UMFPACK_LIB})
else()
- message(FATAL_ERROR "Did not find the umfpack library")
+ message(FATAL_ERROR "Could not find the UMFPACK library")
endif()
+ find_library(BLAS_LIBRARY NAMES "blas" "goto" "mkl"
+ HINTS ENV MKL_LIB
+ DOC "The BLAS library")
+ if(BLAS_LIBRARY)
+ list(APPEND AMDIS_LIBRARIES ${BLAS_LIBRARY})
+ list(APPEND AMDIS_LIBRARY_DIRS ${BLAS_LIBRARY_DIR})
+ else()
+ message(ERROR "Could not find the BLAS library. Please set the variable BLAS_LIBRARY to the blas library with full path")
+ endif()
+
find_library(AMD_LIB amd
- DOC "the AMD library")
- if( AMD_LIB )
+ HINTS ENV LIBRARY_PATH
+ DOC "The AMD library")
+ if(AMD_LIB)
list(APPEND AMDIS_LIBRARIES ${AMD_LIB})
else()
- message(FATAL_ERROR "Did not find the amd library")
+ message(FATAL_ERROR "Could not find the AMD library.")
endif()
- #message("amdis-libs: ${AMDIS_LIBRARIES}")
- list(APPEND AMDIS_LIBRARY_DIRS ${BLAS_LIBRARY_DIR})
endif(AMDIS_NEED_UMFPACK)
@@ -135,13 +108,20 @@ list(APPEND AMDIS_INCLUDE_DIRS ${AMDIS_INCLUDE_DIR}/compositeFEM)
list(APPEND AMDIS_INCLUDE_DIRS ${AMDIS_INCLUDE_DIR}/muparser)
if(${AMDIS_FIND_COMPONENTS} MATCHES umfpack )
- if( NOT AMDIS_NEED_UMFPACK )
- set( AMDIS_umfpack_FOUND FALSE)
- message(SEND_ERROR "the selected amdis was not compiled with umfpack")
+ if(NOT AMDIS_NEED_UMFPACK)
+ set(AMDIS_umfpack_FOUND FALSE)
+ message(SEND_ERROR "The selected amdis was not compiled with umfpack")
endif()
endif()
if(AMDIS_NEED_ZOLTAN)
- find_library(ZOLTAN_LIB zoltan DOC "full path to the zoltan library")
- list(APPEND AMDIS_LIBRARIES ${ZOLTAN_LIB})
+ find_library(ZOLTAN_LIB zoltan
+ HINTS ENV LIBRARY_PATH
+ DOC "full path to the zoltan library")
+
+ if(ZOLTAN_LIB)
+ list(APPEND AMDIS_LIBRARIES ${ZOLTAN_LIB})
+ else()
+ message(FATAL_ERROR "Could not find zoltan library!")
+ endif()
endif()
diff --git a/AMDiS/CMakeLists.txt b/AMDiS/CMakeLists.txt
index 2987385179a44bb9885b0abc494cfd2633f541af..a238f467a810d9cf12c19acad45187d73598536d 100644
--- a/AMDiS/CMakeLists.txt
+++ b/AMDiS/CMakeLists.txt
@@ -246,8 +246,7 @@ endif(ENABLE_PARALLEL_DOMAIN)
if(ENABLE_UMFPACK)
find_file(UMFPACK_H umfpack.h
- PATH ENV CPATH
- HINTS /usr/include /usr/include/suitesparse /usr/include/ufsparse
+ HINTS ENV CPATH /usr/include /usr/include/suitesparse /usr/include/ufsparse
DOC "headerfile umfpack.h for UMFPACK")
if(UMFPACK_H)
get_filename_component(UMFPACK_PATH ${UMFPACK_H} PATH)
diff --git a/AMDiS/src/AdaptInstationary.cc b/AMDiS/src/AdaptInstationary.cc
index b2201e45ac894a71ce16707b43bf38d574665d5d..b1c16f3c9461367f33933df51bcf126e391ab435 100644
--- a/AMDiS/src/AdaptInstationary.cc
+++ b/AMDiS/src/AdaptInstationary.cc
@@ -342,11 +342,6 @@ namespace AMDiS {
}
}
-#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
- MeshDistributor::globalMeshDistributor->exitParallelization();
- PetscFinalize();
-#endif
-
return errorCode;
}
diff --git a/AMDiS/src/AdaptStationary.cc b/AMDiS/src/AdaptStationary.cc
index 964babed0fd76770492846d734c16f58eda7259a..d02b92688427a5be77c08f6c3de070143d4b1e00 100644
--- a/AMDiS/src/AdaptStationary.cc
+++ b/AMDiS/src/AdaptStationary.cc
@@ -77,11 +77,6 @@ namespace AMDiS {
adaptInfo->incSpaceIteration();
}
-#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
- MeshDistributor::globalMeshDistributor->exitParallelization();
- PetscFinalize();
-#endif
-
return 0;
}
diff --git a/AMDiS/src/DOFVector.cc b/AMDiS/src/DOFVector.cc
index 4e68d3a76bb3221f552845a258b79c4431df18e9..001cf56c0923d14918bb5ad1a0d7d5af43f3fd44 100644
--- a/AMDiS/src/DOFVector.cc
+++ b/AMDiS/src/DOFVector.cc
@@ -897,6 +897,11 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localValue = value;
+ MPI::COMM_WORLD.Allreduce(&localValue, &value, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
return value;
}
@@ -939,6 +944,11 @@ namespace AMDiS {
cont = dualTraverse.traverseNext(dualElInfo);
}
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localValue = value;
+ MPI::COMM_WORLD.Allreduce(&localValue, &value, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
return value;
}
@@ -979,6 +989,11 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localValue = value;
+ MPI::COMM_WORLD.Allreduce(&localValue, &value, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
return value;
}
@@ -1014,6 +1029,11 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localValue = value;
+ MPI::COMM_WORLD.Allreduce(&localValue, &value, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
return value;
}
}
diff --git a/AMDiS/src/DOFVector.hh b/AMDiS/src/DOFVector.hh
index d7ce38ff09e645a3248913b9717a43daffc13b8f..6f202d73e486170c577df6516b454c6698235cc5 100644
--- a/AMDiS/src/DOFVector.hh
+++ b/AMDiS/src/DOFVector.hh
@@ -171,9 +171,15 @@ namespace AMDiS {
for (vecIterator.reset(); !vecIterator.end(); ++vecIterator)
nrm += (*vecIterator) * (*vecIterator);
- return(sqrt(nrm));
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localNrm = nrm;
+ MPI::COMM_WORLD.Allreduce(&localNrm, &nrm, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return sqrt(nrm);
}
+
template<typename T>
double DOFVector<T>::squareNrm2() const
{
@@ -187,9 +193,15 @@ namespace AMDiS {
for (vecIterator.reset(); !vecIterator.end(); ++vecIterator)
nrm += (*vecIterator) * (*vecIterator);
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localNrm = nrm;
+ MPI::COMM_WORLD.Allreduce(&localNrm, &nrm, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
return nrm;
}
+
template<typename T>
T DOFVector<T>::asum() const
{
@@ -203,9 +215,15 @@ namespace AMDiS {
for (vecIterator.reset(); !vecIterator.end(); ++vecIterator)
nrm += abs(*vecIterator);
- return(nrm);
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localNrm = nrm;
+ MPI::COMM_WORLD.Allreduce(&localNrm, &nrm, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return nrm;
}
+
template<typename T>
T DOFVector<T>::sum() const
{
@@ -219,9 +237,15 @@ namespace AMDiS {
for (vecIterator.reset(); !vecIterator.end(); ++vecIterator)
nrm += *vecIterator;
- return(nrm);
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localNrm = nrm;
+ MPI::COMM_WORLD.Allreduce(&localNrm, &nrm, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return nrm;
}
+
template<typename T>
void DOFVector<T>::set(T alpha)
{
@@ -268,6 +292,11 @@ namespace AMDiS {
for (vecIterator.reset(), m = *vecIterator; !vecIterator.end(); ++vecIterator)
m = std::min(m, *vecIterator);
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localMin = m;
+ MPI::COMM_WORLD.Allreduce(&localMin, &m, 1, MPI_DOUBLE, MPI_MIN);
+#endif
+
return m;
}
@@ -284,10 +313,15 @@ namespace AMDiS {
for (vecIterator.reset(), m = *vecIterator; !vecIterator.end(); ++vecIterator)
m = std::max(m, *vecIterator);
- return m;
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localMax = m;
+ MPI::COMM_WORLD.Allreduce(&localMax, &m, 1, MPI_DOUBLE, MPI_MAX);
+#endif
+ return m;
}
+
template<typename T>
T DOFVector<T>::absMax() const
{
@@ -310,6 +344,13 @@ namespace AMDiS {
count++;
}
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localSum = m;
+ int localCount = count;
+ MPI::COMM_WORLD.Allreduce(&localSum, &m, 1, MPI_DOUBLE, MPI_SUM);
+ MPI::COMM_WORLD.Allreduce(&localCount, &count, 1, MPI_INT, MPI_SUM);
+#endif
+
return m / count;
}
@@ -365,6 +406,7 @@ namespace AMDiS {
return;
}
+
template<typename T>
int DOFVector<T>::calcMemoryUsage() const
{
@@ -375,6 +417,7 @@ namespace AMDiS {
return result;
}
+
template<typename T>
T DOFVectorBase<T>::evalUh(const DimVec<double>& lambda,
DegreeOfFreedom* dof_indices)
@@ -386,9 +429,15 @@ namespace AMDiS {
for (int i = 0; i < nBasisFcts; i++)
val += (*this)[dof_indices[i]]*(*phi->getPhi(i))(lambda);
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localVal = val;
+ MPI::COMM_WORLD.Allreduce(&localVal, &val, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
return val;
}
+
template<typename T>
void DOFVector<T>::interpol(AbstractFunction<T, WorldVector<double> > *fct)
{
@@ -433,6 +482,7 @@ namespace AMDiS {
}
}
+
template<typename T>
void DOFVector<T>::interpolFct(ElInfo* elinfo)
{
@@ -450,6 +500,7 @@ namespace AMDiS {
(*traverseVector)[myLocalIndices[i]] = inter_val[i];
}
+
template<typename T>
double DOFVector<T>::Int(Quadrature* q) const
{
@@ -483,7 +534,12 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
- return result;
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localResult = result;
+ MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return result;
}
@@ -520,7 +576,12 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
- return result;
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localResult = result;
+ MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return result;
}
@@ -557,7 +618,12 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
- return result;
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localResult = result;
+ MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return result;
}
@@ -602,9 +668,15 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
- return result;
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localResult = result;
+ MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return result;
}
+
template<typename T>
void DOFVector<T>::compressDOFIndexed(int first, int last,
std::vector<DegreeOfFreedom> &newDOF)
@@ -614,6 +686,7 @@ namespace AMDiS {
vec[newDOF[i]] = vec[i];
}
+
template<typename T>
Flag DOFVectorBase<T>::getAssembleFlag()
{
@@ -626,6 +699,7 @@ namespace AMDiS {
return fillFlag;
}
+
template<typename T>
void DOFVectorBase<T>::finishAssembling()
{
@@ -635,6 +709,7 @@ namespace AMDiS {
(*it)->finishAssembling();
}
+
template<typename T>
DOFVector<T>& DOFVector<T>::operator=(const DOFVector<T>& rhs)
{
@@ -664,6 +739,7 @@ namespace AMDiS {
return *this;
}
+
template<typename T>
const DOFVector<T>& operator*=(DOFVector<T>& x, T scal)
{
@@ -703,6 +779,7 @@ namespace AMDiS {
return x;
}
+
template<typename T>
const DOFVector<T>& operator-=(DOFVector<T>& x, const DOFVector<T>& y)
{
@@ -725,6 +802,7 @@ namespace AMDiS {
return x;
}
+
template<typename T>
const DOFVector<T>& operator*=(DOFVector<T>& x, const DOFVector<T>& y)
{
@@ -747,6 +825,7 @@ namespace AMDiS {
return x;
}
+
template<typename T>
T operator*(DOFVector<T>& x, DOFVector<T>& y)
{
@@ -771,6 +850,7 @@ namespace AMDiS {
return dot;
}
+
template<typename T>
void mv(MatrixTranspose transpose, const DOFMatrix &a, const DOFVector<T>&x,
DOFVector<T> &result, bool add)
@@ -806,6 +886,7 @@ namespace AMDiS {
ERROR_EXIT("transpose = %d\n", transpose);
}
+
template<typename T>
void axpy(double alpha, const DOFVector<T>& x, DOFVector<T>& y)
{
@@ -831,6 +912,7 @@ namespace AMDiS {
y[i] += alpha * x[i];
}
+
template<typename T>
const DOFVector<T>& operator*(const DOFVector<T>& v, double d)
{
@@ -838,6 +920,7 @@ namespace AMDiS {
return mult(d, v, result);
}
+
template<typename T>
const DOFVector<T>& operator*(double d, const DOFVector<T>& v)
{
@@ -845,6 +928,7 @@ namespace AMDiS {
return mult(d, v, result);
}
+
template<typename T>
const DOFVector<T>& operator+(const DOFVector<T> &v1 , const DOFVector<T> &v2)
{
@@ -880,6 +964,7 @@ namespace AMDiS {
y[i] = alpha * y[i] + x[i];
}
+
template<typename T>
inline const DOFVector<T>& mult(double scal,
const DOFVector<T>& v,
@@ -894,6 +979,7 @@ namespace AMDiS {
return result;
}
+
template<typename T>
inline const DOFVector<T>& add(const DOFVector<T>& v,
double scal,
@@ -1048,6 +1134,11 @@ namespace AMDiS {
elInfo = stack.traverseNext(elInfo);
}
- return result;
+#ifdef HAVE_PARALLEL_DOMAIN_AMDIS
+ double localResult = result;
+ MPI::COMM_WORLD.Allreduce(&localResult, &result, 1, MPI_DOUBLE, MPI_SUM);
+#endif
+
+ return result;
}
}
diff --git a/AMDiS/src/parallel/MeshDistributor.cc b/AMDiS/src/parallel/MeshDistributor.cc
index e0310499e918a142c15c9736604217f3036c3c19..b55ad5bf09c5a707eb3628fa157d4a05a850b6fe 100644
--- a/AMDiS/src/parallel/MeshDistributor.cc
+++ b/AMDiS/src/parallel/MeshDistributor.cc
@@ -138,12 +138,14 @@ namespace AMDiS {
{
FUNCNAME("MeshDistributor::initParallelization()");
+ if (initialized)
+ return;
+
TEST_EXIT(mpiSize > 1)
("Parallelization does not work with only one process!\n");
TEST_EXIT(feSpace)("No FE space has been defined for the mesh distributor!\n");
TEST_EXIT(mesh)("No mesh has been defined for the mesh distributor!\n");
- TEST_EXIT(!initialized)("MeshDistributor is already initialized!\n");
#ifdef HAVE_ZOLTAN
int a = 0;