Flows: fix bottleneck from construction of identical net detection

Author: N-Maas

mt-kahypar/partition/refinement/flows/flow_refinement_scheduler.cpp

@@ -219,6 +219,10 @@ HyperedgeWeight FlowRefinementScheduler<GraphAndGainTypes>::runFlowSearch(Partit
   auto start = std::chrono::high_resolution_clock::now();
   if ( sub_hg.numNodes() > 0 ) {
     auto partitioned_hg = utils::partitioned_hg_const_cast(phg);
+
+    if (_refiner[refiner_idx] == nullptr) {
+      _refiner[refiner_idx] = constructFlowRefiner();
+    }
     _refiner[refiner_idx]->initialize(partitioned_hg);
     _refiner[refiner_idx]->updateTimeLimit(time_limit);
     MoveSequence sequence = _refiner[refiner_idx]->refine(partitioned_hg, sub_hg, start);
@@ -370,13 +374,6 @@ void FlowRefinementScheduler<GraphAndGainTypes>::initializeImpl(mt_kahypar_parti
       phg.partWeight(i), _context.partition.max_part_weights[i]);
   }
 
-  // Initialize Refiners
-  for (auto& refiner: _refiner) {
-    if ( refiner == nullptr ) {
-      refiner = constructFlowRefiner();
-    }
-  }
-
   // Initialize Quotient Graph and Scheduler
   _stats.reset();
   utils::Timer& timer = utils::Utilities::instance().getTimer(_context.utility_id);

mt-kahypar/partition/refinement/flows/flow_refiner.cpp

@@ -157,10 +157,18 @@ FlowProblem FlowRefiner<GraphAndGainTypes>::constructFlowHypergraph(const Partit
 
   const bool sequential = _context.shared_memory.num_threads == _context.refinement.flows.num_parallel_searches;
   if ( sequential ) {
-    flow_problem = _sequential_construction.constructFlowHypergraph(
+    if ( _sequential_construction == nullptr ) {
+      _sequential_construction = std::make_unique<SequentialConstruction<GraphAndGainTypes>>(
+        _num_hyperedges, _flow_hg, _sequential_hfc, _context);
+    }
+    flow_problem = _sequential_construction->constructFlowHypergraph(
       phg, sub_hg, _block_0, _block_1, _whfc_to_node);
   } else {
-    flow_problem = _parallel_construction.constructFlowHypergraph(
+    if ( _parallel_construction == nullptr ) {
+      _parallel_construction = std::make_unique<ParallelConstruction<GraphAndGainTypes>>(
+        _num_hyperedges, _flow_hg, _parallel_hfc, _context);
+    }
+    flow_problem = _parallel_construction->constructFlowHypergraph(
       phg, sub_hg, _block_0, _block_1, _whfc_to_node);
   }
 

mt-kahypar/partition/refinement/flows/flow_refiner.h

@@ -27,6 +27,8 @@
 
 #pragma once
 
+#include <memory>
+
 #include <tbb/concurrent_vector.h>
 
 #include "WHFC/algorithm/hyperflowcutter.h"
@@ -57,14 +59,15 @@ class FlowRefiner final : public IFlowRefiner {
                        const Context& context) :
     _phg(nullptr),
     _context(context),
+    _num_hyperedges(num_hyperedges),
     _block_0(kInvalidPartition),
     _block_1(kInvalidPartition),
     _flow_hg(),
     _sequential_hfc(_flow_hg, context.partition.seed),
     _parallel_hfc(_flow_hg, context.partition.seed),
     _whfc_to_node(),
-    _sequential_construction(num_hyperedges, _flow_hg, _sequential_hfc, context),
-    _parallel_construction(num_hyperedges, _flow_hg, _parallel_hfc, context) {
+    _sequential_construction(nullptr),
+    _parallel_construction(nullptr) {
       _sequential_hfc.find_most_balanced = _context.refinement.flows.find_most_balanced_cut;
       _sequential_hfc.timer.active = false;
       _sequential_hfc.forceSequential(true);
@@ -111,14 +114,15 @@ class FlowRefiner final : public IFlowRefiner {
   const Context& _context;
   using IFlowRefiner::_time_limit;
 
+  HyperedgeID _num_hyperedges;
   PartitionID _block_0;
   PartitionID _block_1;
   FlowHypergraphBuilder _flow_hg;
   whfc::HyperFlowCutter<whfc::SequentialPushRelabel> _sequential_hfc;
   whfc::HyperFlowCutter<whfc::ParallelPushRelabel> _parallel_hfc;
 
   vec<HypernodeID> _whfc_to_node;
-  SequentialConstruction<GraphAndGainTypes> _sequential_construction;
-  ParallelConstruction<GraphAndGainTypes> _parallel_construction;
+  std::unique_ptr<SequentialConstruction<GraphAndGainTypes>> _sequential_construction;
+  std::unique_ptr<ParallelConstruction<GraphAndGainTypes>> _parallel_construction;
 };
 }  // namespace mt_kahypar

mt-kahypar/partition/refinement/flows/parallel_construction.cpp

@@ -40,9 +40,15 @@ template<typename GraphAndGainTypes>
 typename ParallelConstruction<GraphAndGainTypes>::TmpHyperedge
 ParallelConstruction<GraphAndGainTypes>::DynamicIdenticalNetDetection::get(const size_t he_hash,
                                                                         const vec<whfc::Node>& pins) {
+  if constexpr (PartitionedHypergraph::is_graph) {
+    // there can't be identical nets for graphs
+    return TmpHyperedge { 0, std::numeric_limits<size_t>::max(), whfc::invalidHyperedge };
+  }
+
   const size_t bucket_idx = he_hash % _hash_buckets.size();
   if ( __atomic_load_n(&_hash_buckets[bucket_idx].threshold, __ATOMIC_RELAXED) == _threshold ) {
     // There exists already some hyperedges with the same hash
+    _hash_buckets[bucket_idx].lock.lock();
     for ( const ThresholdHyperedge& tmp : _hash_buckets[bucket_idx].identical_nets ) {
       // Check if there is some hyperedge equal to he
       const TmpHyperedge& tmp_e = tmp.e;
@@ -57,28 +63,33 @@ ParallelConstruction<GraphAndGainTypes>::DynamicIdenticalNetDetection::get(const
           }
         }
         if ( is_identical ) {
+          _hash_buckets[bucket_idx].lock.unlock();
           return tmp_e;
         }
       }
     }
+    _hash_buckets[bucket_idx].lock.unlock();
   }
   return TmpHyperedge { 0, std::numeric_limits<size_t>::max(), whfc::invalidHyperedge };
 }
 
 template<typename GraphAndGainTypes>
 void ParallelConstruction<GraphAndGainTypes>::DynamicIdenticalNetDetection::add(const TmpHyperedge& tmp_he) {
+  if constexpr (PartitionedHypergraph::is_graph) {
+    // there can't be identical nets for graphs
+    return;
+  }
+
   const size_t bucket_idx = tmp_he.hash % _hash_buckets.size();
-  uint32_t expected = __atomic_load_n(&_hash_buckets[bucket_idx].threshold, __ATOMIC_RELAXED);
-  uint32_t desired = _threshold - 1;
-  while ( __atomic_load_n(&_hash_buckets[bucket_idx].threshold, __ATOMIC_RELAXED) < _threshold ) {
-    if ( expected < desired &&
-        __atomic_compare_exchange(&_hash_buckets[bucket_idx].threshold,
-          &expected, &desired, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED) ) {
-      _hash_buckets[bucket_idx].identical_nets.clear();
-      __atomic_store_n(&_hash_buckets[bucket_idx].threshold, _threshold, __ATOMIC_RELAXED);
-    }
+  HashBucket& bucket = _hash_buckets[bucket_idx];
+  bucket.lock.lock();
+  const uint32_t current = __atomic_load_n(&bucket.threshold, __ATOMIC_RELAXED);
+  if (current < _threshold) {
+      bucket.identical_nets.clear();
+      __atomic_store_n(&bucket.threshold, _threshold, __ATOMIC_RELAXED);
   }
-  _hash_buckets[bucket_idx].identical_nets.push_back(ThresholdHyperedge { tmp_he, _threshold });
+  bucket.identical_nets.push_back(ThresholdHyperedge { tmp_he, _threshold });
+  bucket.lock.unlock();
 }
 
 template<typename GraphAndGainTypes>

mt-kahypar/partition/refinement/flows/parallel_construction.h

@@ -32,6 +32,7 @@
 #include "WHFC/algorithm/hyperflowcutter.h"
 #include "WHFC/algorithm/parallel_push_relabel.h"
 
+#include "mt-kahypar/parallel/atomic_wrapper.h"
 #include "mt-kahypar/partition/context.h"
 #include "mt-kahypar/datastructures/sparse_map.h"
 #include "mt-kahypar/datastructures/concurrent_flat_map.h"
@@ -74,8 +75,7 @@ class ParallelConstruction {
       const uint32_t threshold;
     };
 
-    using IdenticalNetVector = tbb::concurrent_vector<
-      ThresholdHyperedge, parallel::zero_allocator<ThresholdHyperedge>>;
+    using IdenticalNetVector = vec<ThresholdHyperedge>;
 
     struct HashBucket {
       HashBucket() :
@@ -84,6 +84,7 @@ class ParallelConstruction {
 
       IdenticalNetVector identical_nets;
       uint32_t threshold;
+      SpinLock lock;
     };
 
    public:
@@ -92,9 +93,11 @@ class ParallelConstruction {
                                           const Context& context) :
       _flow_hg(flow_hg),
       _hash_buckets(),
-      _threshold(2) {
-      _hash_buckets.resize(std::max(UL(1024), num_hyperedges /
-        context.refinement.flows.num_parallel_searches));
+      _threshold(1) {
+      if constexpr (!PartitionedHypergraph::is_graph) {
+        _hash_buckets.resize(std::max(UL(1024), num_hyperedges /
+          std::max(UL(8), 2 * context.refinement.flows.num_parallel_searches)));
+      }
     }
 
     TmpHyperedge get(const size_t he_hash,
@@ -103,7 +106,7 @@ class ParallelConstruction {
     void add(const TmpHyperedge& tmp_he);
 
     void reset() {
-      _threshold += 2;
+      _threshold++;
     }
 
    private:

mt-kahypar/partition/refinement/flows/sequential_construction.cpp

@@ -37,6 +37,11 @@ namespace mt_kahypar {
 template<typename GraphAndGainTypes>
 whfc::Hyperedge SequentialConstruction<GraphAndGainTypes>::DynamicIdenticalNetDetection::add_if_not_contained(
   const whfc::Hyperedge he, const size_t he_hash, const vec<whfc::Node>& pins) {
+  if constexpr (PartitionedHypergraph::is_graph) {
+    // there can't be identical nets for graphs
+    return whfc::invalidHyperedge;
+  }
+
   const size_t bucket_idx = he_hash % _hash_buckets.size();
   if ( _hash_buckets[bucket_idx].threshold == _threshold ) {
     // There exists already some hyperedges with the same hash

mt-kahypar/partition/refinement/flows/sequential_construction.h

@@ -80,8 +80,10 @@ class SequentialConstruction {
       _flow_hg(flow_hg),
       _hash_buckets(),
       _threshold(1) {
-      _hash_buckets.resize(std::max(UL(1024), num_hyperedges /
-        context.refinement.flows.num_parallel_searches));
+      if constexpr (!PartitionedHypergraph::is_graph) {
+        _hash_buckets.resize(std::max(UL(1024), num_hyperedges /
+          std::max(UL(8), 2 * context.refinement.flows.num_parallel_searches)));
+      }
     }
 
     /**