MeshPartitioner.cc 4.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
//
// Software License for AMDiS
//
// Copyright (c) 2010 Dresden University of Technology 
// All rights reserved.
// Authors: Simon Vey, Thomas Witkowski et al.
//
// This file is part of AMDiS
//
// See also license.opensource.txt in the distribution.


#include "parallel/MeshPartitioner.h"
#include "Mesh.h"
#include "Traverse.h"
#include "Serializer.h"

namespace AMDiS {

  void MeshPartitioner::createInitialPartitioning() 
  {
    FUNCNAME("MeshPartitioner::createInitialPartitioning()");

    int mpiRank = mpiComm->Get_rank();
    int mpiSize = mpiComm->Get_size();
    int nLeaves = mesh->getNumberOfLeaves();
    int elPerRank = nLeaves / mpiSize;

    // === Create initial partitioning of the AMDiS mesh. ===

    elementInRank.clear();
32
33

    // Is used in box partitioning mode only. 
34
    map<DofEdge, std::set<int> > vertexElements;
35
36

    TraverseStack stack;
37
38
    ElInfo *elInfo = 
      stack.traverseFirst(mesh, 0, 
39
			  Mesh::CALL_EL_LEVEL | Mesh::FILL_NEIGH | Mesh::FILL_BOUND);
40
    while (elInfo) {
Thomas Witkowski's avatar
Thomas Witkowski committed
41
42
      Element *el = elInfo->getElement();
      int elIndex = el->getIndex();
43

44
45
46
      // === Store for all macro elements the interior neighbours (thus, no ===
      // === periodic neighbours) in the map elNeighbours.                  ===

47
48
49
      for (int i = 0; i < mesh->getGeo(NEIGH); i++)
	if (elInfo->getNeighbour(i) && elInfo->getBoundary(i) == INTERIOR) 
	  elNeighbours[elIndex].push_back(elInfo->getNeighbour(i)->getIndex());
Thomas Witkowski's avatar
Thomas Witkowski committed
50

51
      // === Create initial partitioning. ===
52
      
53
54
55
56
      if (!boxPartitioning) {
	// In standard mode assign to each macro element an arbitrary but unique
	// rank number.
	
Thomas Witkowski's avatar
Thomas Witkowski committed
57
	int elInRank = std::min(elIndex / elPerRank, mpiSize - 1);
58
59
60
61
62
//  	int elInRank = 0;
//  	if (elIndex <= 11 || elIndex >= 28)
//  	  elInRank = 0;
//  	else
//  	  elInRank = 1;
63

Thomas Witkowski's avatar
Thomas Witkowski committed
64
65
	elementInRank[elIndex] = (elInRank == mpiRank);
	partitionMap[elIndex] = elInRank;	
66
67
68
69
      } else {
	// In box partitioning mode, we do the assignment of boxes to ranks later.

	vertexElements[el->getEdge(0)].insert(elIndex);
Thomas Witkowski's avatar
Thomas Witkowski committed
70
      }
71
72
73
      
      elInfo = stack.traverseNext(elInfo);
    }
Thomas Witkowski's avatar
Thomas Witkowski committed
74
75


76
    // === Do initial partitioning in box partitioning mode. ===
Thomas Witkowski's avatar
Thomas Witkowski committed
77
78
79
80

    if (boxPartitioning) {
      TEST_EXIT(mesh->getDim() == 3)("Box partitioning only implemented for 3D!\n");

81
82
83
84

      // === Create boxes: all elements sharing the same 0-edge are assumed ===
      // === to be in the same box.                                         ===

Thomas Witkowski's avatar
Thomas Witkowski committed
85
      int boxCounter = 0;
86
      for (map<DofEdge, std::set<int> >::iterator it = vertexElements.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
87
88
89
90
91
	   it != vertexElements.end(); ++it) {
	TEST_EXIT_DBG(it->second.size() == 6)("Should not happen!\n");
	
	boxSplitting[boxCounter] = it->second;

92
	for (std::set<int>::iterator elIt = it->second.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
93
94
95
96
97
98
	     elIt != it->second.end(); ++elIt)
	  elInBox[*elIt] = boxCounter;

	boxCounter++;
      }

99
100
101

      // === Calculate box neighbourhood relation. ===

Thomas Witkowski's avatar
Thomas Witkowski committed
102
103
104
105
106
107
108
109
110
111
112
113
114
115
      for (map<int, int>::iterator it = elInBox.begin(); it != elInBox.end(); ++it) {
	int elBoxNo = it->second;

	for (vector<int>::iterator neighIt = elNeighbours[it->first].begin();
	     neighIt != elNeighbours[it->first].end(); ++neighIt) {
	  int neighBoxNo = elInBox[*neighIt];

	  if (elBoxNo != neighBoxNo)
	    boxNeighbours[elBoxNo].insert(neighBoxNo);
	}
      }

      MSG("Box partitioning with %d boxes enabled!\n", boxCounter);

116
117
118

      /// === Create initial partitioning of the boxes. ====

Thomas Witkowski's avatar
Thomas Witkowski committed
119
120
      int boxPerRank = boxCounter / mpiSize;

121
      for (map<int, std::set<int> >::iterator it = boxSplitting.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
122
123
124
	   it != boxSplitting.end(); ++it) {
	int boxInRank = std::min(it->first / boxPerRank, mpiSize - 1);

125
	for (std::set<int>::iterator elIt = it->second.begin();
Thomas Witkowski's avatar
Thomas Witkowski committed
126
127
128
129
130
131
	     elIt != it->second.end(); ++elIt) {
	  elementInRank[*elIt] = (boxInRank == mpiRank);
	  partitionMap[*elIt] = boxInRank;
	}
      }
    }
132
133
134
135
136
137
  }


  void MeshPartitioner::serialize(std::ostream &out)
  {
    SerUtil::serialize(out, elementInRank);
Thomas Witkowski's avatar
Thomas Witkowski committed
138
139
140
141
    SerUtil::serialize(out, boxPartitioning);
    SerUtil::serialize(out, boxSplitting);
    SerUtil::serialize(out, boxNeighbours);
    SerUtil::serialize(out, elInBox);
142
143
144
145
146
147
  }


  void MeshPartitioner::deserialize(std::istream &in)
  {
    SerUtil::deserialize(in, elementInRank);
Thomas Witkowski's avatar
Thomas Witkowski committed
148
149
150
151
    SerUtil::deserialize(in, boxPartitioning);
    SerUtil::deserialize(in, boxSplitting);
    SerUtil::deserialize(in, boxNeighbours);
    SerUtil::deserialize(in, elInBox);
152
153
154
  }

}