Merge pull request #2694 from mabel-dev/prefilter-renamed-prune,-stats-added

pruned blob statisitics

Author: joocer

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 1381
+__build__ = 1382
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

opteryx/connectors/capabilities/statistics.py

@@ -44,7 +44,7 @@ def read_blob_statistics(
             self.stats_cache.set(key, cached_stats)
         return cached_stats
 
-    def prefilter_blobs(self, blob_names: list[str], query_statistics, selection) -> list[str]:
+    def prune_blobs(self, blob_names: list[str], query_statistics, selection) -> list[str]:
         new_blob_names = []
 
         for blob_name in blob_names:

opteryx/models/query_statistics.py

@@ -42,6 +42,8 @@ def as_dict(self):
         """
         Return statistics as a dictionary
         """
+        import opteryx
+        
         stats_dict = dict(self._stats)
         for k, v in stats_dict.items():
             # times are recorded in ns but reported in seconds
@@ -53,6 +55,7 @@ def as_dict(self):
         # sort the keys in the dictionary
         stats_dict = {key: stats_dict[key] for key in sorted(stats_dict)}
         # put messages and executed_plan at the end
+        stats_dict["version"] = opteryx.__version__
         stats_dict["messages"] = stats_dict.pop("messages", [])
         stats_dict["executed_plan"] = stats_dict.pop("executed_plan", None)
         return stats_dict

opteryx/operators/async_read_node.py

@@ -106,11 +106,13 @@ def execute(self, morsel, **kwargs) -> Generator:
             predicates=self.predicates,
         )
 
-        if self.predicates and hasattr(reader, "prefilter_blobs"):
+        if self.predicates and hasattr(reader, "prune_blobs"):
             # if we're capturing statistics, we can prefilter the blobs
-            blob_names = reader.prefilter_blobs(
+            start = time.monotonic_ns()
+            blob_names = reader.prune_blobs(
                 blob_names=blob_names, query_statistics=self.statistics, selection=self.predicates
             )
+            self.statistics.time_pruning_blobs += time.monotonic_ns() - start
 
         if len(blob_names) == 0:
             # if we don't have any matching blobs, create an empty dataset