[Refactor] Move scan range filter primitives to storage primitive

be/src/exec/olap_common.h

@@ -34,129 +34,19 @@
 
 #pragma once
 
-#include <boost/container/flat_set.hpp>
-#include <boost/lexical_cast.hpp>
 #include <cstdint>
-#include <map>
+#include <memory>
 #include <sstream>
 #include <string>
 #include <utility>
-#include <variant>
+#include <vector>
 
-#include "base/string/slice.h"
-#include "column/runtime_type_traits.h"
 #include "exec/olap_utils.h"
-#include "exec/scan_node.h"
-#include "gen_cpp/PlanNodes_types.h"
-#include "gutil/stl_util.h"
-#include "gutil/strings/substitute.h"
+#include "storage/primitive/column_value_range.h"
 #include "storage/primitive/olap_tuple.h"
-#include "types/date_value.h"
-#include "types/datetime_value.h"
-#include "types/timestamp_value.h"
 
 namespace starrocks {
 
-// There are two types of value range: Fixed Value Range and Range Value Range
-// I know "Range Value Range" sounds bad, but it's hard to turn over the de facto.
-// Fixed Value Range means discrete values in the set, like "IN (1,2,3)"
-// Range Value Range means range values like ">= 10 && <= 20"
-/**
- * @brief Column's value range
- **/
-template <class T>
-class ColumnValueRange {
-public:
-    using RangeValueType = T;
-    using ValuesContainer = boost::container::flat_set<T>;
-    using iterator_type = typename ValuesContainer::iterator;
-
-    ColumnValueRange();
-    ColumnValueRange(std::string col_name, LogicalType type, T min, T max);
-    ColumnValueRange(std::string col_name, LogicalType type, T type_min, T type_max, T min, T max);
-
-    Status add_fixed_values(SQLFilterOp op, const ValuesContainer& values);
-
-    Status add_range(SQLFilterOp op, T value);
-
-    void set_precision(int precision);
-
-    void set_scale(int scale);
-
-    int precision() const;
-
-    int scale() const;
-
-    bool is_fixed_value_range() const;
-
-    bool is_empty_value_range() const;
-
-    bool is_full_value_range() const;
-
-    bool is_init_state() const { return _is_init_state; }
-
-    bool is_fixed_value_convertible() const;
-
-    SQLFilterOp fixed_value_operator() const { return _fixed_op; }
-
-    bool is_range_value_convertible() const;
-
-    size_t get_convertible_fixed_value_size() const;
-
-    void convert_to_fixed_value();
-
-    void convert_to_range_value();
-
-    const ValuesContainer& get_fixed_value_set() const { return _fixed_values; }
-
-    T get_range_max_value() const { return _high_value; }
-
-    T get_range_min_value() const { return _low_value; }
-
-    bool is_low_value_mininum() const { return _low_value == _type_min; }
-
-    bool is_high_value_maximum() const { return _high_value == _type_max; }
-
-    bool is_begin_include() const { return _low_op == FILTER_LARGER_OR_EQUAL; }
-
-    bool is_end_include() const { return _high_op == FILTER_LESS_OR_EQUAL; }
-
-    LogicalType type() const { return _column_type; }
-
-    size_t get_fixed_value_size() const { return _fixed_values.size(); }
-
-    void set_index_filter_only(bool is_index_only) { _is_index_filter_only = is_index_only; }
-
-    template <typename ConditionType, bool Negative = false>
-    void to_olap_filter(std::vector<ConditionType>& filters);
-
-    template <typename ConditionType>
-    ConditionType to_olap_not_null_filter() const;
-
-    void clear();
-    void clear_to_empty();
-
-private:
-    std::string _column_name;
-    LogicalType _column_type{TYPE_UNKNOWN}; // Column type (eg: TINYINT,SMALLINT,INT,BIGINT)
-    int _precision;                         // casting a decimalv3-typed value into string need precision
-    int _scale;                             // casting a decimalv3-typed value into string need scale
-    T _type_min;                            // Column type's min value
-    T _type_max;                            // Column type's max value
-    T _low_value;                           // Column's low value, closed interval at left
-    T _high_value;                          // Column's high value, open interval at right
-    SQLFilterOp _low_op;
-    SQLFilterOp _high_op;
-    ValuesContainer _fixed_values; // Column's fixed values
-    SQLFilterOp _fixed_op;
-    // Whether this condition only used to filter index, not filter chunk row in storage engine
-    bool _is_index_filter_only = false;
-    // ColumnValueRange don't call add_range or add_fixed_values
-    bool _is_init_state = true;
-
-    bool _empty_range = false;
-};
-
 class OlapScanKeys {
 public:
     OlapScanKeys() = default;
@@ -206,20 +96,4 @@ class OlapScanKeys {
     bool _is_convertible{true};
 };
 
-// clang-format off
-using ColumnValueRangeType =  std::variant<
-        ColumnValueRange<int8_t>,
-        ColumnValueRange<uint8_t>,  // vectorized boolean
-        ColumnValueRange<int16_t>,
-        ColumnValueRange<int32_t>,
-        ColumnValueRange<int64_t>,
-        ColumnValueRange<__int128>,
-        ColumnValueRange<int256_t>,
-        ColumnValueRange<Slice>,
-        ColumnValueRange<DecimalV2Value>,
-        ColumnValueRange<bool>,
-        ColumnValueRange<DateValue>,
-        ColumnValueRange<TimestampValue>>;
-// clang-format on
-
 } // namespace starrocks

be/src/exec/olap_scan_prepare.h

@@ -19,8 +19,8 @@
 #include "exec/olap_common.h"
 #include "exprs/expr.h"
 #include "exprs/expr_context.h"
-#include "filter_condition.h"
 #include "runtime/descriptors.h"
+#include "storage/primitive/filter_condition.h"
 #include "storage/primitive/predicate_tree/predicate_tree_fwd.h"
 #include "storage/primitive/predicate_tree_params.h"
 #include "storage/runtime_filter_predicate.h"

be/src/exec/olap_utils.h

@@ -36,14 +36,8 @@
 
 #include <cmath>
 
-#include "column/runtime_type_traits.h"
-#include "common/logging.h"
-#include "gen_cpp/Exprs_types.h"
-#include "gen_cpp/Opcodes_types.h"
+#include "storage/primitive/filter_op.h"
 #include "storage/primitive/olap_tuple.h"
-#include "types/datetime_value.h"
-#include "types/logical_type.h"
-#include "types/logical_type_infra.h"
 
 namespace starrocks {
 
@@ -66,75 +60,4 @@ typedef struct OlapScanRange {
     OlapTuple end_scan_range;
 } OlapScanRange;
 
-enum SQLFilterOp {
-    FILTER_LARGER = 0,
-    FILTER_LARGER_OR_EQUAL = 1,
-    FILTER_LESS = 2,
-    FILTER_LESS_OR_EQUAL = 3,
-    FILTER_IN = 4,
-    FILTER_NOT_IN = 5
-};
-
-inline SQLFilterOp to_olap_filter_type(TExprOpcode::type type, bool opposite) {
-    switch (type) {
-    case TExprOpcode::LT:
-        return opposite ? FILTER_LARGER : FILTER_LESS;
-    case TExprOpcode::LE:
-        //TODO: Datetime may be truncated to a date column, so we convert LT to LE,
-        // for example: '2010-01-01 00:00:01' will be truncate to '2010-01-01'
-        return opposite ? FILTER_LARGER_OR_EQUAL : FILTER_LESS_OR_EQUAL;
-    case TExprOpcode::GT:
-        return opposite ? FILTER_LESS : FILTER_LARGER;
-
-    case TExprOpcode::GE:
-        return opposite ? FILTER_LESS_OR_EQUAL : FILTER_LARGER_OR_EQUAL;
-
-    case TExprOpcode::EQ:
-        return opposite ? FILTER_NOT_IN : FILTER_IN;
-
-    case TExprOpcode::NE:
-        return opposite ? FILTER_IN : FILTER_NOT_IN;
-
-    case TExprOpcode::EQ_FOR_NULL:
-        return FILTER_IN;
-
-    default:
-        VLOG(2) << "TExprOpcode: " << type;
-        DCHECK(false);
-    }
-
-    return FILTER_IN;
-}
-
-inline SQLFilterOp invert_olap_filter_type(const SQLFilterOp op) {
-    switch (op) {
-    case FILTER_LARGER:
-        return FILTER_LESS_OR_EQUAL;
-    case FILTER_LARGER_OR_EQUAL:
-        return FILTER_LESS;
-    case FILTER_LESS:
-        return FILTER_LARGER_OR_EQUAL;
-    case FILTER_LESS_OR_EQUAL:
-        return FILTER_LARGER;
-    case FILTER_IN:
-        return FILTER_NOT_IN;
-    case FILTER_NOT_IN:
-        return FILTER_IN;
-    default:
-        VLOG(2) << "Unkown SQLFilterOp when inverting it: " << op;
-        DCHECK(false);
-    }
-    return FILTER_IN;
-}
-
-template <bool Negative>
-SQLFilterOp to_olap_filter_type(TExprOpcode::type type, bool opposite) {
-    const auto op = to_olap_filter_type(type, opposite);
-    if constexpr (Negative) {
-        return invert_olap_filter_type(op);
-    } else {
-        return op;
-    }
-}
-
 } // namespace starrocks

be/src/exec/pipeline/set/except_context.cpp

@@ -14,7 +14,9 @@
 
 #include "exec/pipeline/set/except_context.h"
 
+#include "exprs/expr.h"
 #include "runtime/current_thread.h"
+#include "runtime/descriptors.h"
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 

be/src/exec/pipeline/set/except_context.h

@@ -23,11 +23,11 @@
 #include "column/runtime_type_traits.h"
 #include "common/statusor.h"
 #include "exec/except_hash_set.h"
-#include "exec/olap_common.h"
 #include "exec/pipeline/context_with_dependency.h"
 #include "exec/pipeline/primitives/pipeline_observer.h"
 #include "exprs/expr_context.h"
 #include "gutil/casts.h"
+#include "runtime/descriptors_fwd.h"
 #include "runtime/mem_pool.h"
 
 namespace starrocks::pipeline {

be/src/exec/pipeline/set/intersect_context.cpp

@@ -14,7 +14,9 @@
 
 #include "exec/pipeline/set/intersect_context.h"
 
+#include "exprs/expr.h"
 #include "runtime/current_thread.h"
+#include "runtime/descriptors.h"
 #include "runtime/runtime_state.h"
 
 namespace starrocks::pipeline {

be/src/exec/pipeline/set/intersect_context.h

@@ -23,11 +23,11 @@
 #include "column/runtime_type_traits.h"
 #include "common/statusor.h"
 #include "exec/intersect_hash_set.h"
-#include "exec/olap_common.h"
 #include "exec/pipeline/context_with_dependency.h"
 #include "exec/pipeline/primitives/pipeline_observer.h"
 #include "exprs/expr_context.h"
 #include "gutil/casts.h"
+#include "runtime/descriptors_fwd.h"
 #include "runtime/mem_pool.h"
 
 namespace starrocks::pipeline {

be/src/formats/parquet/column_reader.cpp

@@ -29,8 +29,8 @@
 #include "formats/parquet/scalar_column_reader.h"
 #include "formats/utils.h"
 #include "gen_cpp/parquet_types.h"
-#include "storage/column_or_predicate.h"
 #include "storage/column_predicate_factory.h"
+#include "storage/primitive/column_or_predicate.h"
 
 namespace starrocks::parquet {
 

be/src/formats/parquet/file_reader.cpp

@@ -36,6 +36,7 @@
 #include "gen_cpp/parquet_types.h"
 #include "gutil/casts.h"
 #include "gutil/strings/substitute.h"
+#include "storage/runtime_range_pruner.hpp"
 
 namespace starrocks::parquet {
 

be/src/formats/parquet/file_reader.h

@@ -30,7 +30,6 @@
 #include "formats/parquet/meta_helper.h"
 #include "formats/parquet/metadata.h"
 #include "gen_cpp/parquet_types.h"
-#include "storage/runtime_range_pruner.hpp"
 
 namespace tparquet {
 class ColumnMetaData;
@@ -44,6 +43,7 @@ struct HdfsScannerContext;
 class BlockCache;
 class StoragePageCache;
 class SlotDescriptor;
+class RuntimeScanRangePruner;
 
 namespace parquet {
 struct ParquetField;

be/src/storage/CMakeLists.txt

@@ -22,6 +22,9 @@ add_subdirectory(rowset)
 ADD_BE_LIB(StoragePrimitive
     primitive/aggregate_type.cpp
     primitive/chunk_iterator.cpp
+    primitive/column_and_predicate.cpp
+    primitive/column_or_predicate.cpp
+    primitive/column_value_range.cpp
     primitive/edit_version.cpp
     primitive/empty_iterator.cpp
     primitive/flat_json_config.cpp
@@ -177,8 +180,6 @@ set(STORAGE_FILES
     column_in_predicate.cpp
     column_not_in_predicate.cpp
     column_null_predicate.cpp
-    column_or_predicate.cpp
-    column_and_predicate.cpp
     column_expr_predicate.cpp
     conjunctive_predicates.cpp
     runtime_filter_predicate.cpp

be/src/storage/predicate_parser.cpp

@@ -27,8 +27,9 @@
 
 namespace starrocks {
 
-ColumnPredicate* PredicateParser::create_column_predicate(const TCondition& condition, TypeInfoPtr& type_info,
-                                                          ColumnId index) {
+namespace {
+
+ColumnPredicate* create_column_predicate(const TCondition& condition, TypeInfoPtr& type_info, ColumnId index) {
     ColumnPredicate* pred = nullptr;
     if ((condition.condition_op == "*=" || condition.condition_op == "=") && condition.condition_values.size() == 1) {
         pred = new_column_eq_predicate(type_info, index, condition.condition_values[0]);
@@ -60,8 +61,7 @@ ColumnPredicate* PredicateParser::create_column_predicate(const TCondition& cond
     return pred;
 }
 
-ColumnPredicate* PredicateParser::create_column_predicate(const GeneralCondition& condition, TypeInfoPtr& type_info,
-                                                          ColumnId index) {
+ColumnPredicate* create_column_predicate(const GeneralCondition& condition, TypeInfoPtr& type_info, ColumnId index) {
     ColumnPredicate* pred = nullptr;
     if ((condition.condition_op == "*=" || condition.condition_op == "=") && condition.condition_values.size() == 1) {
         pred = new_column_eq_predicate_from_datum(type_info, index, condition.condition_values[0]);
@@ -92,6 +92,8 @@ ColumnPredicate* PredicateParser::create_column_predicate(const GeneralCondition
     return pred;
 }
 
+} // namespace
+
 bool OlapPredicateParser::can_pushdown(const ColumnPredicate* predicate) const {
     RETURN_IF(predicate->column_id() >= _schema->num_columns(), false);
     const TabletColumn& column = _schema->column(predicate->column_id());