Fixed regression tests

Author: joocer

opteryx/__version__.py

@@ -1,4 +1,4 @@
-__build__ = 1390
+__build__ = 1392
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

opteryx/utils/file_decoders.py

@@ -270,7 +270,7 @@ def parquet_decoder(
 
         for column in schema.names:
             column_index = schema.get_field_index(column)
-            
+
             for rg_index in range(num_row_groups):
                 column_chunk = metadata.row_group(rg_index).column(column_index)
 

tests/catalog/test_catalog_iceberg.py

@@ -10,6 +10,7 @@
 import opteryx
 from opteryx.connectors import DiskConnector
 from opteryx.connectors import IcebergConnector
+from opteryx.compiled.structures.relation_statistics import to_int
 
 
 # this is how we get the raw list of files for the scan
@@ -73,16 +74,16 @@ def test_iceberg_get_statistics_manual():
             stats.update_upper(column_names[k], IcebergConnector.decode_iceberg_value(v, column_types[k]))
 
     assert stats.record_count == 100000
-    assert stats.lower_bounds["followers"] == 0
-    assert stats.upper_bounds["followers"] == 8266250
-    assert stats.lower_bounds["user_name"] == ""
-    assert stats.upper_bounds["user_name"] == "🫖🔫"
-    assert stats.lower_bounds["tweet_id"] == 1346604539013705728
-    assert stats.upper_bounds["tweet_id"] == 1346615999009755142
-    assert stats.lower_bounds["text"] == "!! PLEASE STOP A"
-    assert stats.upper_bounds["text"] == "🪶Cultural approq"
-    assert stats.lower_bounds["timestamp"] == "2021-01-05T23:48"
-    assert stats.upper_bounds["timestamp"] == "2021-01-06T00:35"
+    assert stats.lower_bounds[b"followers"] == 0
+    assert stats.upper_bounds[b"followers"] == 8266250
+    assert stats.lower_bounds[b"user_name"] == to_int("")
+    assert stats.upper_bounds[b"user_name"] == to_int("🫖🔫")
+    assert stats.lower_bounds[b"tweet_id"] == to_int(1346604539013705728)
+    assert stats.upper_bounds[b"tweet_id"] == to_int(1346615999009755142)
+    assert stats.lower_bounds[b"text"] == to_int("!! PLEASE STOP A")
+    assert stats.upper_bounds[b"text"] == to_int("🪶Cultural approq")
+    assert stats.lower_bounds[b"timestamp"] == to_int("2021-01-05T23:48")
+    assert stats.upper_bounds[b"timestamp"] == to_int("2021-01-06T00:35")
 
 @skip_if(is_arm() or is_windows() or is_mac())
 def test_iceberg_connector():
@@ -106,16 +107,16 @@ def test_iceberg_get_stats_tweets():
     stats = connector.relation_statistics
 
     assert stats.record_count == 100000
-    assert stats.lower_bounds["followers"] == 0
-    assert stats.upper_bounds["followers"] == 8266250
-    assert stats.lower_bounds["user_name"] == ""
-    assert stats.upper_bounds["user_name"] == "🫖🔫"
-    assert stats.lower_bounds["tweet_id"] == 1346604539013705728
-    assert stats.upper_bounds["tweet_id"] == 1346615999009755142
-    assert stats.lower_bounds["text"] == "!! PLEASE STOP A"
-    assert stats.upper_bounds["text"] == "🪶Cultural approq"
-    assert stats.lower_bounds["timestamp"] == "2021-01-05T23:48"
-    assert stats.upper_bounds["timestamp"] == "2021-01-06T00:35"
+    assert stats.lower_bounds[b"followers"] == 0
+    assert stats.upper_bounds[b"followers"] == 8266250
+    assert stats.lower_bounds[b"user_name"] == to_int("")
+    assert stats.upper_bounds[b"user_name"] == to_int("🫖🔫")
+    assert stats.lower_bounds[b"tweet_id"] == to_int(1346604539013705728)
+    assert stats.upper_bounds[b"tweet_id"] == to_int(1346615999009755142)
+    assert stats.lower_bounds[b"text"] == to_int("!! PLEASE STOP A")
+    assert stats.upper_bounds[b"text"] == to_int("🪶Cultural approq")
+    assert stats.lower_bounds[b"timestamp"] == to_int("2021-01-05T23:48")
+    assert stats.upper_bounds[b"timestamp"] == to_int("2021-01-06T00:35")
 
 @skip_if(is_arm() or is_windows() or is_mac())
 def test_iceberg_get_stats_missions():
@@ -130,16 +131,16 @@ def test_iceberg_get_stats_missions():
     stats = connector.relation_statistics
 
     assert stats.record_count == 100000
-    assert stats.lower_bounds["followers"] == 0
-    assert stats.upper_bounds["followers"] == 8266250
-    assert stats.lower_bounds["user_name"] == ""
-    assert stats.upper_bounds["user_name"] == "🫖🔫"
-    assert stats.lower_bounds["tweet_id"] == 1346604539013705728
-    assert stats.upper_bounds["tweet_id"] == 1346615999009755142
-    assert stats.lower_bounds["text"] == "!! PLEASE STOP A"
-    assert stats.upper_bounds["text"] == "🪶Cultural approq"
-    assert stats.lower_bounds["timestamp"] == "2021-01-05T23:48"
-    assert stats.upper_bounds["timestamp"] == "2021-01-06T00:35"
+    assert stats.lower_bounds[b"followers"] == 0
+    assert stats.upper_bounds[b"followers"] == 8266250
+    assert stats.lower_bounds[b"user_name"] == to_int("")
+    assert stats.upper_bounds[b"user_name"] == to_int("🫖🔫")
+    assert stats.lower_bounds[b"tweet_id"] == to_int(1346604539013705728)
+    assert stats.upper_bounds[b"tweet_id"] == to_int(1346615999009755142)
+    assert stats.lower_bounds[b"text"] == to_int("!! PLEASE STOP A")
+    assert stats.upper_bounds[b"text"] == to_int("🪶Cultural approq")
+    assert stats.lower_bounds[b"timestamp"] == to_int("2021-01-05T23:48")
+    assert stats.upper_bounds[b"timestamp"] == to_int("2021-01-06T00:35")
 
 @skip_if(is_arm() or is_windows() or is_mac())
 def test_iceberg_get_stats_remote():
@@ -165,18 +166,18 @@ def test_iceberg_get_stats_remote():
     stats = connector.relation_statistics
 
     assert stats.record_count == 9
-    assert stats.lower_bounds["id"] == 1, stats.lower_bounds["id"]
-    assert stats.upper_bounds["id"] == 9, stats.upper_bounds["id"]
-    assert stats.lower_bounds["name"] == "Earth", stats.lower_bounds["name"]
-    assert stats.upper_bounds["name"] == "Venus", stats.upper_bounds["name"]
-    assert stats.lower_bounds["mass"] == 0.0146, stats.lower_bounds["mass"]
-    assert stats.upper_bounds["mass"] == 1898.0, stats.upper_bounds["mass"]
-    assert stats.lower_bounds["diameter"] == 2370, stats.lower_bounds["diameter"]
-    assert stats.upper_bounds["diameter"] == 142984, stats.upper_bounds["diameter"]
-    assert stats.lower_bounds["gravity"] == Decimal("0.7"), stats.lower_bounds["gravity"]
-    assert stats.upper_bounds["gravity"] == Decimal("23.1"), stats.upper_bounds["gravity"]
-    assert stats.lower_bounds["surfacePressure"] == 0.0, stats.lower_bounds["surfacePressure"]
-    assert stats.upper_bounds["surfacePressure"] == 92.0, stats.upper_bounds["surfacePressure"]
+    assert stats.lower_bounds[b"id"] == 1, stats.lower_bounds[b"id"]
+    assert stats.upper_bounds[b"id"] == 9, stats.upper_bounds[b"id"]
+    assert stats.lower_bounds[b"name"] == to_int("Earth"), stats.lower_bounds[b"name"]
+    assert stats.upper_bounds[b"name"] == to_int("Venus"), stats.upper_bounds[b"name"]
+    assert stats.lower_bounds[b"mass"] == to_int(0.0146), stats.lower_bounds[b"mass"]
+    assert stats.upper_bounds[b"mass"] == to_int(1898.0), stats.upper_bounds[b"mass"]
+    assert stats.lower_bounds[b"diameter"] == 2370, stats.lower_bounds[b"diameter"]
+    assert stats.upper_bounds[b"diameter"] == 142984, stats.upper_bounds[b"diameter"]
+    assert stats.lower_bounds[b"gravity"] == to_int(Decimal("0.7")), stats.lower_bounds[b"gravity"]
+    assert stats.upper_bounds[b"gravity"] == to_int(Decimal("23.1")), stats.upper_bounds[b"gravity"]
+    assert stats.lower_bounds[b"surfacePressure"] == to_int(0.0), stats.lower_bounds[b"surfacePressure"]
+    assert stats.upper_bounds[b"surfacePressure"] == to_int(92.0), stats.upper_bounds[b"surfacePressure"]
 
 
 @skip_if(is_arm() or is_windows() or is_mac())

tests/query_planner/test_read_parquet_statistics.py

@@ -4,6 +4,7 @@
 sys.path.insert(1, os.path.join(sys.path[0], "../.."))
 
 from opteryx.utils.file_decoders import parquet_decoder
+from opteryx.compiled.structures.relation_statistics import to_int
 
 def test_read_statistics_tweets():
     with open("testdata/flat/formats/parquet/tweets.parquet", "rb") as f:
@@ -13,13 +14,13 @@ def test_read_statistics_tweets():
 
     assert stats.record_count == 100000, stats.record_count
 
-    assert stats.lower_bounds["tweet_id"] == 1346604539013705728
-    assert stats.upper_bounds["tweet_id"] == 1346615999009755142
-    assert stats.null_count["tweet_id"] == 0
+    assert stats.lower_bounds[b"tweet_id"] == to_int(1346604539013705728)
+    assert stats.upper_bounds[b"tweet_id"] == to_int(1346615999009755142)
+    assert stats.null_count.get(b"tweet_id", 0) == 0
 
-    assert stats.lower_bounds["is_quoting"] == 28466111963996160
-    assert stats.upper_bounds["is_quoting"] == 1346615755694104578, stats.upper_bounds["is_quoting"]
-    assert stats.null_count["is_quoting"] == 85598
+    assert stats.lower_bounds[b"is_quoting"] == to_int(28466111963996160)
+    assert stats.upper_bounds[b"is_quoting"] == to_int(1346615755694104578)
+    assert stats.null_count[b"is_quoting"] == 85598
 
 
 def test_read_statistics_planets():
@@ -30,32 +31,22 @@ def test_read_statistics_planets():
 
     assert stats.record_count == 9
 
-    assert stats.lower_bounds["id"] == 1
-    assert stats.upper_bounds["id"] == 9
-    assert stats.null_count["id"] == 0
+    assert stats.lower_bounds[b"id"] == 1
+    assert stats.upper_bounds[b"id"] == 9
+    assert stats.null_count.get(b"id", 0) == 0
 
-    assert stats.lower_bounds["name"] == "Earth"
-    assert stats.upper_bounds["name"] == "Venus"
-    assert stats.null_count["name"] == 0
+    assert stats.lower_bounds[b"name"] == to_int("Earth")
+    assert stats.upper_bounds[b"name"] == to_int("Venus")
+    assert stats.null_count.get(b"name", 0) == 0
 
-    assert stats.lower_bounds["surfacePressure"] == 0.0
-    assert stats.upper_bounds["surfacePressure"] == 92.0
+    assert stats.lower_bounds[b"surfacePressure"] == to_int(0.0)
+    assert stats.upper_bounds[b"surfacePressure"] == to_int(92.0)
 
 
 
-def test_read_statistics():
-    with open("testdata/astronauts/astronauts.parquet", "rb") as f:
-        data = f.read()
-
-        stats = parquet_decoder(data, just_statistics=True)
-
-    print(stats.record_count)
-    print(stats.lower_bounds)
-    print(stats.upper_bounds)
-    print(stats.null_count)
-
-
 if __name__ == "__main__":  # pragma: no cover
     from tests.tools import run_tests
 
+    test_read_statistics_planets()
+
     run_tests()