Skip to content

Commit a042620

Browse files
T3C42T3C42
committed
Add Competitors and improve streaming
1 parent 341abc6 commit a042620

6 files changed

Lines changed: 384 additions & 10 deletions

File tree

‎mt-kahypar/dynamic/competitors/hermes.h‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#pragma once
2+
3+
#include <mt-kahypar/dynamic/dynamic_strategy.h>
4+
5+
namespace mt_kahypar::dyn {
6+
7+
class Hermes : public DynamicStrategy {
8+
9+
private:
10+
11+
PartitionID assign_node_first_free_partition(const HypernodeID& hn) {
12+
for (PartitionID p = 0; p < context.partition.k; ++p) {
13+
if (partitioned_hypergraph_m.partWeight(p) + hypergraph_m.nodeWeight(hn) <=
14+
context.partition.max_part_weights[p]) {
15+
partitioned_hypergraph_m.addNode(hn, p);
16+
return p;
17+
}
18+
}
19+
// if no partition could accommodate the node put in the first
20+
partitioned_hypergraph_m.addNode(hn, 0);
21+
return 0;
22+
}
23+
24+
PartitionID getTargetPart(const HypernodeID& hn, const size_t stage) {
25+
const PartitionID current_part = partitioned_hypergraph_m.partID(hn);
26+
ASSERT(current_part != kInvalidPartition);
27+
size_t max_gain = 0;
28+
PartitionID target_part = current_part;
29+
for ( PartitionID p = 0; p < context.partition.k; ++p ) {
30+
if (p != current_part) {
31+
if (stage == 1 && p > current_part || stage == 2 && p < current_part) {
32+
size_t gain = 0;
33+
for ( const HyperedgeID& he : hypergraph_m.incidentEdges(hn) ) {
34+
if (partitioned_hypergraph_m.pinCountInPart(he, p) == 0) {
35+
gain -= hypergraph_m.edgeWeight(he);
36+
}
37+
if (partitioned_hypergraph_m.pinCountInPart(he, current_part) == 1) {
38+
gain += hypergraph_m.edgeWeight(he);
39+
}
40+
}
41+
if (gain > max_gain && partitioned_hypergraph_m.partWeight(p) + hypergraph_m.nodeWeight(hn) <=
42+
context.partition.max_part_weights[p]) {
43+
max_gain = gain;
44+
target_part = p;
45+
}
46+
}
47+
}
48+
}
49+
return target_part;
50+
}
51+
52+
public:
53+
54+
Hermes(ds::MutableHypergraph& hypergraph_m, Context& context)
55+
: DynamicStrategy(hypergraph_m, context) {}
56+
57+
MutablePartitionedHypergraph& init() override {
58+
partitioned_hypergraph_m = partition_hypergraph_km1(hypergraph_m, context);
59+
ASSERT(partitioned_hypergraph_m.checkAllConnectivitySets());
60+
return partitioned_hypergraph_m;
61+
}
62+
63+
void partition(Change& change, size_t changes_size) override {
64+
(void) changes_size;
65+
ASSERT(metrics::isBalanced(partitioned_hypergraph_m, context));
66+
// ASSERT(partitioned_hypergraph_m.checkAllConnectivitySets());
67+
68+
// Removals are not supported in streaming mode
69+
70+
ASSERT(change.removed_nodes.empty());
71+
ASSERT(change.removed_edges.empty());
72+
ASSERT(change.removed_pins.empty());
73+
74+
for (const HypernodeID& hn : change.added_nodes) {
75+
const HypernodeID new_hn = hypergraph_m.addHypernode({}, 1);
76+
(void) new_hn;
77+
ASSERT(hn == new_hn);
78+
updateMaxPartWeight(context, hypergraph_m);
79+
const PartitionID assigned_part = assign_node_first_free_partition(hn);
80+
(void) assigned_part;
81+
ASSERT(assigned_part != kInvalidPartition);
82+
}
83+
84+
for (const HyperedgeID& he : change.added_edges) {
85+
hypergraph_m.addHyperedge({}, 1);
86+
partitioned_hypergraph_m.addEdge(he);
87+
}
88+
89+
for (const auto& [node, edge] : change.added_pins)
90+
{
91+
hypergraph_m.addPin(edge, node);
92+
partitioned_hypergraph_m.incrementPinCountOfBlockWrapper(edge, partitioned_hypergraph_m.partID(node));
93+
}
94+
95+
// for (const HypernodeID& hn : change.added_nodes)
96+
for (const HypernodeID& hn : hypergraph_m.nodes())
97+
{
98+
for (size_t stage = 1; stage <= 2; ++stage)
99+
{
100+
const PartitionID target_part = getTargetPart(hn, stage);
101+
if (target_part != partitioned_hypergraph_m.partID(hn)) {
102+
partitioned_hypergraph_m.changeNodePart(hn, partitioned_hypergraph_m.partID(hn), target_part);
103+
}
104+
}
105+
}
106+
}
107+
};
108+
}

‎mt-kahypar/dynamic/competitors/leopard.h‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
#pragma once
2+
3+
#include <mt-kahypar/dynamic/dynamic_strategy.h>
4+
5+
namespace mt_kahypar::dyn {
6+
7+
class Leopard : public DynamicStrategy {
8+
9+
private:
10+
11+
vec<size_t> skippedComps; // number of consecutive times a node was not examined for reassignment
12+
double skipping_threshold = 0.1; // threshold for examining a node for reassignment as experimentally determined in the leopard paper
13+
14+
PartitionID assign_node_first_free_partition(const HypernodeID& hn) {
15+
for (PartitionID p = 0; p < context.partition.k; ++p) {
16+
if (partitioned_hypergraph_m.partWeight(p) + hypergraph_m.nodeWeight(hn) <=
17+
context.partition.max_part_weights[p]) {
18+
partitioned_hypergraph_m.addNode(hn, p);
19+
return p;
20+
}
21+
}
22+
// if no partition could accommodate the node put in the first
23+
partitioned_hypergraph_m.addNode(hn, 0);
24+
return 0;
25+
}
26+
27+
28+
PartitionID getTargetPartFennel(const HypernodeID& hn) {
29+
double fennel_alpha = sqrt(sqrt(context.partition.k) * hypergraph_m.initialNumEdges() / pow(hypergraph_m.initialNumNodes(),1.5));
30+
const PartitionID current_part = partitioned_hypergraph_m.partID(hn);
31+
ASSERT(current_part != kInvalidPartition);
32+
std::vector block_connectivities(context.partition.k, std::make_tuple(0,0));
33+
for ( PartitionID p = 0; p < context.partition.k; ++p ) {
34+
block_connectivities[p] = std::make_tuple(0, p);
35+
}
36+
// compute neighbor count in each block
37+
for ( const HyperedgeID& he : hypergraph_m.incidentEdges(hn) ) {
38+
for ( const HypernodeID& hn2 : hypergraph_m.pins(he) ) {
39+
block_connectivities[partitioned_hypergraph_m.partID(hn2)] = std::make_tuple(std::get<0>(block_connectivities[partitioned_hypergraph_m.partID(hn2)]) + 1, partitioned_hypergraph_m.partID(hn2));
40+
}
41+
}
42+
std::vector<std::tuple<double, PartitionID>> block_scores(context.partition.k, std::make_tuple(0,0));
43+
for ( PartitionID p = 0; p < context.partition.k; ++p )
44+
{
45+
constexpr double fennel_gamma = 1.5;
46+
double score = std::get<0>(block_connectivities[p]) - fennel_alpha * (fennel_gamma / 2) * pow(partitioned_hypergraph_m.partWeight(p), fennel_gamma - 1);
47+
block_scores[p] = std::make_tuple(score, p);
48+
}
49+
50+
// sort block_scores in descending order
51+
std::sort(block_scores.begin(), block_scores.end(), [](const auto& a, const auto& b) {
52+
return std::get<0>(a) > std::get<0>(b);
53+
});
54+
55+
// return the block with the highest score that doesn't violate max_part_weights (imbalance)
56+
for (const auto& block_score : block_scores)
57+
{
58+
if (std::get<1>(block_score) != current_part)
59+
{
60+
if (partitioned_hypergraph_m.partWeight(std::get<1>(block_score)) + hypergraph_m.nodeWeight(hn) <=
61+
context.partition.max_part_weights[std::get<1>(block_score)])
62+
{
63+
return std::get<1>(block_score);
64+
}
65+
}
66+
}
67+
return current_part;
68+
}
69+
70+
bool toExamineOrNot(const HypernodeID& hn, size_t threshold) {
71+
size_t neighbors = 0;
72+
for (const HyperedgeID& he : hypergraph_m.incidentEdges(hn)) {
73+
for (const HypernodeID& hn2 : hypergraph_m.pins(he)) {
74+
if (hn2 != hn) {
75+
neighbors++;
76+
}
77+
}
78+
}
79+
if (neighbors == 0) {
80+
return false;
81+
}
82+
if ((skippedComps[hn] + 1) / neighbors >= threshold) {
83+
skippedComps[hn] = 0;
84+
return true;
85+
} else {
86+
skippedComps[hn]++;
87+
return false;
88+
}
89+
}
90+
91+
public:
92+
93+
Leopard(ds::MutableHypergraph& hypergraph_m, Context& context)
94+
: DynamicStrategy(hypergraph_m, context) {}
95+
96+
MutablePartitionedHypergraph& init() override {
97+
partitioned_hypergraph_m = partition_hypergraph_km1(hypergraph_m, context);
98+
ASSERT(partitioned_hypergraph_m.checkAllConnectivitySets());
99+
for (HypernodeID hn = 0; hn < hypergraph_m.initialNumNodes(); ++hn) {
100+
skippedComps.push_back(0);
101+
}
102+
return partitioned_hypergraph_m;
103+
}
104+
105+
void partition(Change& change, size_t changes_size) override {
106+
(void) changes_size;
107+
ASSERT(metrics::isBalanced(partitioned_hypergraph_m, context));
108+
// ASSERT(partitioned_hypergraph_m.checkAllConnectivitySets());
109+
110+
// Removals are not supported in streaming mode
111+
112+
ASSERT(change.removed_nodes.empty());
113+
ASSERT(change.removed_edges.empty());
114+
ASSERT(change.removed_pins.empty());
115+
116+
for (const HypernodeID& hn : change.added_nodes) {
117+
const HypernodeID new_hn = hypergraph_m.addHypernode({}, 1);
118+
(void) new_hn;
119+
ASSERT(hn == new_hn);
120+
updateMaxPartWeight(context, hypergraph_m);
121+
const PartitionID assigned_part = assign_node_first_free_partition(hn);
122+
(void) assigned_part;
123+
ASSERT(assigned_part != kInvalidPartition);
124+
skippedComps.push_back(0);
125+
}
126+
127+
for (const HyperedgeID& he : change.added_edges) {
128+
hypergraph_m.addHyperedge({}, 1);
129+
partitioned_hypergraph_m.addEdge(he);
130+
}
131+
132+
for (const auto& [node, edge] : change.added_pins)
133+
{
134+
hypergraph_m.addPin(edge, node);
135+
partitioned_hypergraph_m.incrementPinCountOfBlockWrapper(edge, partitioned_hypergraph_m.partID(node));
136+
}
137+
138+
std::vector<HypernodeID> nodes_to_examine;
139+
for (const HyperedgeID& he : change.added_edges)
140+
{
141+
for (const HypernodeID& hn : hypergraph_m.pins(he))
142+
{
143+
nodes_to_examine.push_back(hn);
144+
}
145+
}
146+
147+
size_t examined_nodes = 1;
148+
149+
while (!nodes_to_examine.empty() && examined_nodes < 10)
150+
{
151+
const HypernodeID hn = nodes_to_examine.back();
152+
nodes_to_examine.pop_back();
153+
examined_nodes++;
154+
for (size_t stage = 1; stage <= 2; ++stage)
155+
{
156+
const PartitionID target_part = getTargetPartFennel(hn);
157+
if (target_part != partitioned_hypergraph_m.partID(hn)) {
158+
partitioned_hypergraph_m.changeNodePart(hn, partitioned_hypergraph_m.partID(hn), target_part);
159+
for (const HyperedgeID& he : hypergraph_m.incidentEdges(hn))
160+
{
161+
for (const HypernodeID& hn2 : hypergraph_m.pins(he))
162+
{
163+
if (hn2 != hn && toExamineOrNot(hn2, skipping_threshold)) {
164+
nodes_to_examine.push_back(hn2);
165+
}
166+
}
167+
}
168+
}
169+
}
170+
}
171+
}
172+
};
173+
}

‎mt-kahypar/dynamic/dynamic_io.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ namespace mt_kahypar::dyn {
4848
return 0; // Could not determine RAM usage
4949
}
5050

51-
inline void print_progress_bar(size_t i, size_t total, size_t km1, double imbalance) {
51+
inline void print_progress_bar(size_t i, size_t total, size_t km1, double imbalance, double cut) {
5252
// clear the line
5353
std::cout << "\r\033[K" << std::flush;
5454
std::string output = "";
55-
output += "km1: " + std::to_string(km1) + ", imb: " + std::to_string(imbalance);
55+
output += "km1: " + std::to_string(km1) + ", cut:" + std::to_string(cut) + ", imb: " + std::to_string(imbalance);
5656
output += " [";
5757
for (size_t j = 0; j < 50; ++j) {
5858
if (j < i * 50 / total) {
@@ -230,7 +230,8 @@ inline std::vector<HypernodeID> parseIDs(const std::string& line) {
230230
file.close();
231231

232232
if (!context.dynamic.server) {
233-
print_progress_bar(i, max_changes, km1, imbalance);
233+
double cut = (km1 / static_cast<double>(hypergraph.initialNumEdges())) * 100;
234+
print_progress_bar(i, max_changes, km1, imbalance, cut);
234235
}
235236

236237
}

‎mt-kahypar/dynamic/dynamic_partitioner.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
#include "mt-kahypar/dynamic/strategies/localFM_rebalance_vcycle.h"
55
#include "mt-kahypar/dynamic/strategies/repartition.h"
66
#include "mt-kahypar/dynamic/strategies/streaming.h"
7+
#include "mt-kahypar/dynamic/strategies/hashing.h"
8+
#include "mt-kahypar/dynamic/competitors/hermes.h"
9+
#include "mt-kahypar/dynamic/competitors/leopard.h"
710
#include "mt-kahypar/partition/registries/registry.h"
811

912
namespace mt_kahypar::dyn {
@@ -52,8 +55,14 @@ namespace mt_kahypar::dyn {
5255
strategy = new LocalFMRebalanceVCycleV4(hypergraph_m, context);
5356
} else if (context.dynamic.strategy == "streaming") {
5457
strategy = new Streaming(hypergraph_m, context);
58+
} else if (context.dynamic.strategy == "hashing") {
59+
strategy = new Hashing(hypergraph_m, context);
5560
} else if (context.dynamic.strategy == "repartition") {
5661
strategy = new Repartition(hypergraph_m, context);
62+
} else if (context.dynamic.strategy == "hermes") {
63+
strategy = new Hermes(hypergraph_m, context);
64+
} else if (context.dynamic.strategy == "leopard") {
65+
strategy = new Leopard(hypergraph_m, context);
5766
} else {
5867
throw std::runtime_error("Unknown dynamic strategy: " + context.dynamic.strategy);
5968
}

0 commit comments

Comments
 (0)