Implemented mask()

Author: hosseinmoein

benchmarks/polars_performance.py

@@ -30,7 +30,7 @@
 third = datetime.datetime.now()
 
 df3 = df.filter(pl.col("log_normal") > 8)
-print(f"Number of rows after select: {df3.select(pl.count()).item()}")
+print(f"Number of rows after select: {df3.select(pl.len()).item()}")
 fourth = datetime.datetime.now()
 
 # df4 = df.sort(["log_normal", "exponential"]);

docs/HTML/DataFrame.html

@@ -333,6 +333,10 @@ <H2 ID="2"><font color="blue">API Reference with code samples <font size="+4">&#
       <td title="Calculates  K-Nearest Neighbors (KNN) algorithm"><a href="https://htmlpreview.github.io/?https://github.com/hosseinmoein/DataFrame/blob/master/docs/HTML/knn.html">knn</a>()</td>
     </tr>
 
+    <tr class="item" onmouseover="this.style.backgroundColor='#ffff66';" onmouseout="this.style.backgroundColor='#d4e3e5';">
+      <td title="General masking function"><a href="https://htmlpreview.github.io/?https://github.com/hosseinmoein/DataFrame/blob/master/docs/HTML/mask.html">mask</a>()</td>
+    </tr>
+
     <tr class="item" onmouseover="this.style.backgroundColor='#ffff66';" onmouseout="this.style.backgroundColor='#d4e3e5';">
       <td title="Returns the Markov Chains stationary distributions"><a href="https://htmlpreview.github.io/?https://github.com/hosseinmoein/DataFrame/blob/master/docs/HTML/MC_station_dist.html">MC_station_dist</a>()</td>
     </tr>

docs/HTML/mask.html

@@ -3744,6 +3744,29 @@ class   DataFrame : public ThreadGranularity {
     [[nodiscard]] DataFrame<I, HeteroVector<std::size_t(H::align_value)>>
     duplication_mask(bool include_index, bool binary = false) const;
 
+    // This is the generalization of a masking function. Every element of the
+    // given column will be passed to the mfunc and the output of mfunc will
+    // be stored in the result vector. A typical example would be a Boolean
+    // result vector, although the default result is a vector of char.
+    // That’s because std::vector<bool> has an odd implementation that I don’t
+    // like.
+    //
+    // NOTE: Type MT must have default constructor
+    // NOTE: mfunc must be stateless, because it is used in multithreading
+    //
+    // T:
+    //   Type of the "new index" column
+    // MT:
+    //   Masking function return type
+    // col_name:
+    //   Name of the given column
+    // mfunc:
+    //   The masking function
+    //
+    template<typename T, typename MT = char>
+    [[nodiscard]] StlVecType<MT>
+    mask(const char *col_name, std::function<MT(const T &val)> &&mfunc) const;
+
     // It behaves like get_data(), but it returns a View.
     // A view is a DataFrame that is a reference to the original DataFrame.
     // So if you modify anything in the view the original DataFrame will

include/DataFrame/DataFrame.h

@@ -1479,6 +1479,71 @@ using fft_v = FastFourierTransVisitor<T, I, A>;
 
 // ----------------------------------------------------------------------------
 
+/*
+template<std::signed_integral T, typename I = unsigned long, std::size_t A = 0>
+struct  NumTheoreticTransVisitor  {
+
+public:
+
+    DEFINE_VISIT_BASIC_TYPES_2
+
+public:
+
+    // Inverse = false — Turns a vector of coefficients into its NTT form
+    //                   (evaluation at roots of unity).
+    // Inverse = true — Turns back from NTT domain to original coefficients.
+    //
+    //
+    // Vectors will be resized to the smallest power of 2 equal or larger than
+    // vector size. They will be appended with zeros.
+    //
+    template <typename K, typename H>
+    inline void
+    operator() (const K &idx_begin, const K &idx_end,
+                const H &column_begin, const H &column_end)  {
+
+
+    }
+
+    // Multiply two polynomials
+    // {1, 2, 3} represents 1 + 2x + 3x^2
+    // {4, 5, 6} represents 4 + 5x + 6x^2
+    //
+    // Vectors will be resized to the smallest power of 2 equal or larger than
+    // vector size. They will be appended with zeros.
+    //
+    template <typename K, typename H>
+    inline void
+    operator() (const K &idx_begin, const K &idx_end,
+                const H &column1_begin, const H &column1_end,
+                const H &column2_begin, const H &column2_end)  {
+
+
+    }
+
+    inline void pre ()  {  }
+    inline void post ()  {  }
+
+    DEFINE_RESULT
+
+    explicit
+    NumTheoreticTransVisitor(bool inverse = false)
+        : inverse_(inverse),
+          thread_level_(ThreadGranularity::get_thread_level())  {   }
+
+private:
+
+    const bool      inverse_;
+    const long      thread_level_;
+    result_type     result_ {  };
+};
+
+template<std::signed_integral T, typename I = unsigned long, std::size_t A = 0>
+using ntt_v = NumTheoreticTransVisitor<T, I, A>;
+*/
+
+// ----------------------------------------------------------------------------
+
 template<arithmetic T, typename I = unsigned long, std::size_t A = 0>
 struct  EntropyVisitor  {
 

include/DataFrame/DataFrameMLVisitors.h

@@ -382,6 +382,40 @@ duplication_mask (bool include_index, bool binary) const  {
 
 // ----------------------------------------------------------------------------
 
+template<typename I, typename H>
+template<typename T, typename MT>
+typename DataFrame<I, H>::StlVecType<MT> DataFrame<I, H>::
+mask(const char *col_name, std::function<MT(const T &val)> &&mfunc) const  {
+
+    using res_t = StlVecType<MT>;
+
+    const auto      &column = get_column<T>(col_name);
+    const size_type col_s = column.size();
+    res_t           result(col_s);
+    auto            lbd =
+        [&column = std::as_const(column), &result, &mfunc]
+        (size_type begin, size_type end) -> void  {
+            for (auto i = begin; i < end; ++i)
+                result[i] = mfunc(column[i]);
+        };
+    const auto      thread_level =
+        (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level();
+
+    if (thread_level > 2)  {
+        auto    futuers = thr_pool_.parallel_loop(size_type(0), col_s,
+                                                  std::move(lbd));
+
+        for (auto &fut : futuers)  fut.get();
+    }
+    else  {
+        lbd(size_type(0), col_s);
+    }
+
+    return (result);
+}
+
+// ----------------------------------------------------------------------------
+
 template<typename I, typename H>
 template<typename T, typename ... Ts>
 DataFrame<I, HeteroVector<std::size_t(H::align_value)>> DataFrame<I, H>::

include/DataFrame/Internals/DataFrame_get.tcc

@@ -3531,6 +3531,61 @@ static void test_MannWhitneyUTestVisitor()  {
 
 // ----------------------------------------------------------------------------
 
+static void test_mask()  {
+
+    std::cout << "\nTesting mask( ) ..." << std::endl;
+
+    StrDataFrame    ibm;
+
+    try  {
+        ibm.read("IBM.csv", io_format::csv2);
+    }
+    catch (const DataFrameError &ex)  {
+        std::cout << ex.what() << std::endl;
+        ::exit(-1);
+    }
+
+    ibm.load_column("close_mask",
+                    ibm.mask<double, char>("IBM_Close",
+                                           [](const double &close) -> char {
+                                               return (close > 180.0);
+                                           })
+                    );
+    ibm.load_column("close_diff",
+                    ibm.mask<double, double>("IBM_Close",
+                                             [](const double &close) -> double {
+                                                 return (180.0 - close);
+                                             })
+                    );
+
+    const auto  &close_mask = ibm.get_column<char>("close_mask");
+    const auto  &close_diff = ibm.get_column<double>("close_diff");
+
+    assert(close_mask[0] == 0);
+    assert(close_mask[10] == 0);
+    assert(close_mask[247] == 0);
+    assert(close_mask[1180] == 0);
+    assert(close_mask[2761] == 1);
+    assert(close_mask[2806] == 1);
+    assert(close_mask[3372] == 1);
+    assert(close_mask[4106] == 1);
+    assert(close_mask[4966] == 0);
+    assert(close_mask[5030] == 0);
+
+    assert((std::fabs(close_diff[0] - 81.4375) < 0.0001));
+    assert((std::fabs(close_diff[10] - 80.625) < 0.0001));
+    assert((std::fabs(close_diff[247] - 71.93) < 0.0001));
+    assert((std::fabs(close_diff[1180] - 98.19) < 0.0001));
+    assert((std::fabs(close_diff[2761] - -0.360001) < 0.000001));
+    assert((std::fabs(close_diff[2806] - -6.17999) < 0.0001));
+    assert((std::fabs(close_diff[3372] - -13.55) < 0.0001));
+    assert((std::fabs(close_diff[4106] - -0.529999) < 0.000001));
+    assert((std::fabs(close_diff[4966] - 57.06) < 0.0001));
+    assert((std::fabs(close_diff[5030] - 68.34) < 0.0001));
+}
+
+// ----------------------------------------------------------------------------
+
 int main(int, char *[]) {
 
     MyDataFrame::set_optimum_thread_level();
@@ -3592,6 +3647,7 @@ int main(int, char *[]) {
     test_detect_and_change();
     test_KolmoSmirnovTestVisitor();
     test_MannWhitneyUTestVisitor();
+    test_mask();
 
     return (0);
 }