Merge branch 'master' into pack_per_net_bb_state

Author: soheilshahrouz

libs/librrgraph/src/base/rr_graph_storage.cpp

@@ -5,6 +5,7 @@
 #include "vtr_assert.h"
 #include "vtr_error.h"
 #include "librrgraph_types.h"
+#include "vtr_sort.h"
 #include "vtr_util.h"
 
 #include <algorithm>
@@ -194,23 +195,32 @@ void t_rr_graph_storage::init_fan_in() {
     }
 }
 
-namespace {
-/// Functor for sorting edges according to destination node's ID.
-class edge_compare_dest_node {
-public:
-    edge_compare_dest_node(const t_rr_graph_storage& rr_graph_storage) : rr_graph_storage_(rr_graph_storage) {}
+void t_rr_graph_storage::apply_edge_permutation(const std::vector<RREdgeId>& edge_indices) {
+    // Generic lambda that allocates a 'vec'-sized new vector with all elements set to default value,
+    // then builds the new vector to have rearranged elements from 'vec' and finally move the new vector
+    // to replace vec. Essentially does a permutation on vec based on edge_indices.
+    auto array_rearrange = [&edge_indices](auto& vec, auto default_value) {
+        // Since vec could have any type, we need to figure out its type to allocate new_vec.
+        // The scary std::remove_reference stuff does exactly that. This does nothing other than building a new 'vec' sized vector.
+        typename std::remove_reference<decltype(vec)>::type new_vec(vec.size(), default_value);
 
-    bool operator()(RREdgeId lhs, RREdgeId rhs) const {
-        RRNodeId lhs_dest_node = rr_graph_storage_.edge_sink_node(lhs);
-        RRNodeId rhs_dest_node = rr_graph_storage_.edge_sink_node(rhs);
+        size_t new_index = 0;
+        for (RREdgeId edge_index : edge_indices) {
+            RREdgeId new_edge_index = RREdgeId(new_index);
+            new_vec[new_edge_index] = vec[edge_index];
 
-        return lhs_dest_node < rhs_dest_node;
-    }
+            new_index++;
+        }
+        VTR_ASSERT(new_index == vec.size());
+
+        vec = std::move(new_vec);
+    };
 
-private:
-    const t_rr_graph_storage& rr_graph_storage_;
-};
-} // namespace
+    array_rearrange(edge_src_node_, RRNodeId::INVALID());
+    array_rearrange(edge_dest_node_, RRNodeId::INVALID());
+    array_rearrange(edge_switch_, LIBRRGRAPH_UNDEFINED_VAL);
+    array_rearrange(edge_remapped_, false);
+}
 
 size_t t_rr_graph_storage::count_rr_switches(const std::vector<t_arch_switch_inf>& arch_switch_inf,
                                              t_arch_switch_fanin& arch_switch_fanins) {
@@ -222,7 +232,7 @@ size_t t_rr_graph_storage::count_rr_switches(const std::vector<t_arch_switch_inf
 
     // Sort by destination node to collect per node/per switch fan in values
     // This sort is safe to do because partition_edges() has not been invoked yet.
-    sort_edges(edge_compare_dest_node(*this));
+    sort_edges_by_keys(vtr::sort_key(node_storage_.size(), [&](RREdgeId e) { return edge_dest_node_[e]; }));
 
     // Collect the fan-in per switch type for each node in the graph
     // Record the unique switch type/fanin combinations
@@ -318,35 +328,6 @@ void t_rr_graph_storage::mark_edges_as_rr_switch_ids() {
     remapped_edges_ = true;
 }
 
-namespace{
-/// Functor for sorting edges according to source node, with configurable edges coming first
-class edge_compare_src_node_and_configurable_first {
-  public:
-    edge_compare_src_node_and_configurable_first(const vtr::vector<RRSwitchId, t_rr_switch_inf>& rr_switch_inf, const t_rr_graph_storage& rr_graph_storage)
-        : rr_switch_inf_(rr_switch_inf),
-          rr_graph_storage_(rr_graph_storage) {}
-
-    bool operator()(RREdgeId lhs, RREdgeId rhs) const {
-
-        RRNodeId lhs_dest_node = rr_graph_storage_.edge_sink_node(lhs);
-        RRNodeId lhs_src_node = rr_graph_storage_.edge_source_node(lhs);
-        RRSwitchId lhs_switch_type = RRSwitchId(rr_graph_storage_.edge_switch(lhs));
-        bool lhs_is_configurable = rr_switch_inf_[lhs_switch_type].configurable();
-
-        RRNodeId rhs_dest_node = rr_graph_storage_.edge_sink_node(rhs);
-        RRNodeId rhs_src_node = rr_graph_storage_.edge_source_node(rhs);
-        RRSwitchId rhs_switch_type = RRSwitchId(rr_graph_storage_.edge_switch(rhs));
-        bool rhs_is_configurable = rr_switch_inf_[rhs_switch_type].configurable();
-
-        return std::make_tuple(lhs_src_node, !lhs_is_configurable, lhs_dest_node, lhs_switch_type) < std::make_tuple(rhs_src_node, !rhs_is_configurable, rhs_dest_node, rhs_switch_type);
-    }
-
-  private:
-    const vtr::vector<RRSwitchId, t_rr_switch_inf>& rr_switch_inf_;
-    const t_rr_graph_storage& rr_graph_storage_;
-};
-} // namespace
-
 void t_rr_graph_storage::partition_edges(const vtr::vector<RRSwitchId, t_rr_switch_inf>& rr_switches) {
     if (partitioned_) {
         return;
@@ -359,7 +340,11 @@ void t_rr_graph_storage::partition_edges(const vtr::vector<RRSwitchId, t_rr_swit
     //    by assign_first_edges()
     //  - Edges within a source node have the configurable edges before the
     //    non-configurable edges.
-    sort_edges(edge_compare_src_node_and_configurable_first(rr_switches, *this));
+    // Keys are listed from the most significant to the least significant.
+    sort_edges_by_keys(vtr::sort_key(node_storage_.size(), [&](RREdgeId e) { return edge_src_node_[e]; }),
+                       vtr::sort_key(2, [&](RREdgeId e) { return !rr_switches[RRSwitchId(edge_switch_[e])].configurable(); }),
+                       vtr::sort_key(node_storage_.size(), [&](RREdgeId e) { return edge_dest_node_[e]; }),
+                       vtr::sort_key(rr_switches.size(), [&](RREdgeId e) { return edge_switch_[e]; }));
 
     partitioned_ = true;
 

libs/librrgraph/src/base/rr_graph_storage.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <algorithm>
 #include <bitset>
 
 #include "librrgraph_types.h"
@@ -15,6 +14,7 @@
 #include "rr_switch.h"
 #include "vtr_log.h"
 #include "vtr_memory.h"
+#include "vtr_sort.h"
 #include "vtr_strong_id_range.h"
 #include "vtr_array_view.h"
 #include <numeric>
@@ -886,48 +886,42 @@ class t_rr_graph_storage {
     /** @brief Validate that edge data is partitioned correctly.*/
     bool validate_node(RRNodeId node_id, const vtr::vector<RRSwitchId, t_rr_switch_inf>& rr_switches) const;
     bool validate(const vtr::vector<RRSwitchId, t_rr_switch_inf>& rr_switches) const;
-    
+
     /**
-     * @brief Sorts edges according to comparison_function. This is an expensive method that builds the edge array from scratch
-     * and invalidates all the RREdgeIds. This is not an inplace sort, and it is very expensive.
-     * You should not be calling this method more than once or twice in the entire program, definitely do not use it in a hot loop.
-     * @tparam t_comp_func callable object with two size_t arguments. See 'edge_compare_dest_node' for example.
-     * @param comparison_function Comparison function to order edges with.
+     * @brief Sorts edges by one or more small integer keys.
+     *
+     * The result is the same as a stable sort by the tuple of keys, ties keep their
+     * current order, but it is computed with stable counting sort passes instead of a
+     * comparison sort.
+     *
+     * This is an expensive method that rebuilds the edge arrays from scratch and
+     * invalidates all RREdgeIds. It should only be called a few times while the graph
+     * is being built, never in a hot loop.
+     *
+     * Example, sorting by destination node:
+     *
+     *     sort_edges_by_keys(vtr::sort_key(num_nodes, [&](RREdgeId e) { return edge_dest_node_[e]; }));
+     *
+     * Example, sorting by source node, and by destination node among edges that share
+     * a source node:
+     *
+     *     sort_edges_by_keys(vtr::sort_key(num_nodes, [&](RREdgeId e) { return edge_src_node_[e]; }),
+     *                        vtr::sort_key(num_nodes, [&](RREdgeId e) { return edge_dest_node_[e]; }));
+     *
+     * @param keys  One vtr::sort_key per sort criterion, listed from the most significant
+     *              to the least significant. Each is built with vtr::sort_key(num_keys, key_of),
+     *              where key_of maps an RREdgeId to a value smaller than num_keys.
      */
-    template <typename t_comp_func>
-    void sort_edges(t_comp_func comparison_function) {
-
+    template <typename... KeyFns>
+    void sort_edges_by_keys(const vtr::sort_key<KeyFns>&... keys) {
         size_t num_edges = edge_src_node_.size();
         vtr::StrongIdRange<RREdgeId> edge_range(RREdgeId(0), RREdgeId(num_edges));
-        std::vector<RREdgeId> edge_indices(edge_range.begin(), edge_range.end());
 
-        std::stable_sort(edge_indices.begin(), edge_indices.end(), comparison_function);
-        
-        // Generic lambda that allocates a 'vec'-sized new vector with all elements set to default value,
-        // then builds the new vector to have rearranged elements from 'vec' and finally move the new vector
-        // to replace vec. Essentially does a permutation on vec based on edge_indices.
-        auto array_rearrange = [&edge_indices] (auto& vec, auto default_value) {
-
-            // Since vec could have any type, we need to figure out it's type to allocate new_vec.
-            // The scary std::remove_reference stuff does exactly that. This does nothing other than building a new 'vec' sized vector.
-            typename std::remove_reference<decltype(vec)>::type new_vec(vec.size(), default_value);
-
-            size_t new_index = 0;
-            for (RREdgeId edge_index : edge_indices) {
-                RREdgeId new_edge_index = RREdgeId(new_index);
-                new_vec[new_edge_index] = vec[edge_index];
-
-                new_index++;
-            }
-            VTR_ASSERT(new_index == vec.size());
+        // Sort the edge ids 0..num_edges-1 by the keys directly into edge_indices
+        std::vector<RREdgeId> edge_indices(num_edges);
+        vtr::stable_radix_sort(edge_range.begin(), edge_range.end(), edge_indices, keys...);
 
-            vec = std::move(new_vec);
-        };
-
-        array_rearrange(edge_src_node_, RRNodeId::INVALID());
-        array_rearrange(edge_dest_node_, RRNodeId::INVALID());
-        array_rearrange(edge_switch_, LIBRRGRAPH_UNDEFINED_VAL);
-        array_rearrange(edge_remapped_, false);
+        apply_edge_permutation(edge_indices);
     }
 
     /******************
@@ -979,6 +973,12 @@ class t_rr_graph_storage {
      */
     void assign_first_edges();
 
+    /**
+     * @brief Rearranges every edge array so that the new edge i is the old edge edge_indices[i].
+     * Invalidates all RREdgeIds held elsewhere.
+     */
+    void apply_edge_permutation(const std::vector<RREdgeId>& edge_indices);
+
     /** @brief Verify that first_edge_ array correctly partitions rr edge data. */
     bool verify_first_edges() const;
 

libs/librrgraph/src/io/rr_graph_uxsdcxx_serializer.h

@@ -1147,31 +1147,31 @@ class RrGraphSerializer final : public uxsd::RrGraphBase<RrGraphContextTypes> {
         return nullptr;
     }
     inline void finish_rr_graph_rr_edges(void*& /*ctx*/) final {
+        // The sink node and the switch id of every edge must be checked before the edges are
+        // partitioned. partition_edges() sorts the edges by those two values, so an out of range
+        // value read from the file would index outside the node and switch arrays.
+        // Only the source node is checked as the edges are read, in add_rr_edges_edge().
+        for (RREdgeId edge_id : rr_nodes_->all_edges()) {
+            size_t sink_node = size_t(rr_nodes_->edge_sink_node(edge_id));
+            size_t switch_id = size_t(rr_nodes_->edge_switch(edge_id));
+            if (sink_node >= rr_nodes_->size()) {
+                report_error(
+                    "sink_node %zu is larger than rr_nodes.size() %zu",
+                    sink_node, rr_nodes_->size());
+            }
+
+            if (switch_id >= rr_switch_inf_->size()) {
+                report_error(
+                    "switch_id %zu is larger than num_rr_switches %zu",
+                    switch_id, rr_switch_inf_->size());
+            }
+        }
+
         // Partition the rr graph edges for efficient access to
         // configurable/non-configurable edge subsets. Must be done after RR
         // switches have been allocated.
         rr_graph_builder_->mark_edges_as_rr_switch_ids();
         rr_graph_builder_->partition_edges();
-
-        for (size_t source_node = 0; source_node < rr_nodes_->size(); ++source_node) {
-            int num_edges = rr_nodes_->num_edges(RRNodeId(source_node));
-            for (int iconn = 0; iconn < num_edges; ++iconn) {
-                size_t sink_node = size_t(rr_nodes_->edge_sink_node(RRNodeId(source_node), iconn));
-                size_t switch_id = rr_nodes_->edge_switch(RRNodeId(source_node), iconn);
-                if (sink_node >= rr_nodes_->size()) {
-                    report_error(
-                        "sink_node %zu is larger than rr_nodes.size() %zu",
-                        sink_node, rr_nodes_->size());
-                }
-
-                if (switch_id >= rr_switch_inf_->size()) {
-                    report_error(
-                        "switch_id %zu is larger than num_rr_switches %zu",
-                        switch_id, rr_switch_inf_->size());
-                }
-
-            }
-        }
     }
 
     inline EdgeWalker get_rr_graph_rr_edges(void*& /*ctx*/) final {