Merge pull request #2943 from mabel-dev/#2942

BLOB -> VARBINARY

Author: joocer

opteryx/__version__.py

@@ -1,9 +1,9 @@
 # THIS FILE IS AUTOMATICALLY UPDATED DURING THE BUILD PROCESS
 # DO NOT EDIT THIS FILE DIRECTLY
 
-__build__ = 1894
+__build__ = 1895
 __author__ = "@joocer"
-__version__ = "0.26.2-beta.1894"
+__version__ = "0.26.2-beta.1895"
 
 # Store the version here so:
 # 1) we don't load dependencies by storing it in __init__.py

opteryx/functions/__init__.py

@@ -494,6 +494,8 @@ def sleep(x):
     "INT": "INTEGER",  # remove 0.27.0
     "GET": None,  # remove 0.28.0
     "SEARCH": None,  # remove 0.28.0
+    "BLOB": "VARBINARY",  # remove 0.29.0
+    "TRY_BLOB": "TRY_VARBINARY",  # remove 0.29.0
 }
 
 # fmt:off
@@ -526,11 +528,13 @@ def sleep(x):
     "DATE": (lambda x: compute.cast(x, pyarrow.date32()), "DATE", 1.0),
     "PASSTHRU": (lambda x: x, "VARIANT", 1.0),
     "BLOB": (cast_to_blob, "BLOB", 1.0),
+    "VARBINARY": (cast_to_blob, "BLOB", 1.0),
     "TRY_ARRAY": (other_functions.array_cast_safe, "VARIANT", 1.0),
     "TRY_TIMESTAMP": (try_cast("TIMESTAMP"), "TIMESTAMP", 1.0),
     "TRY_BOOLEAN": (try_cast("BOOLEAN"), "BOOLEAN", 1.0),
     "TRY_VARCHAR": (try_cast("VARCHAR"), "VARCHAR", 1.0),
     "TRY_BLOB": (try_cast("BLOB"), "BLOB", 1.0),
+    "TRY_VARBINARY": (try_cast("BLOB"), "BLOB", 1.0),
     "TRY_INTEGER": (try_cast("INTEGER"), "INTEGER", 1.0),
     "TRY_DECIMAL": (try_cast("DECIMAL"), "DECIMAL", 1.0),
     "TRY_DOUBLE": (try_cast("DOUBLE"), "DOUBLE", 1.0),

opteryx/operators/exit_node.py

@@ -17,6 +17,7 @@
 This node doesn't do any calculations, it is a pure Projection.
 """
 
+import pyarrow
 from orso.types import OrsoTypes
 from pyarrow import Table
 
@@ -76,7 +77,6 @@ def execute(self, morsel: Table, **kwargs) -> Table:
         # Exit doesn't return EOS
         if morsel == EOS:
             if not self.at_least_one:
-                import pyarrow
                 from orso.schema import RelationSchema
                 from orso.schema import convert_orso_schema_to_arrow_schema
 
@@ -119,9 +119,22 @@ def execute(self, morsel: Table, **kwargs) -> Table:
         morsel = morsel.select(self.final_columns)
 
         for index, column in enumerate(self.columns):
-            if column.schema_column.type == OrsoTypes.INTERVAL:
-                converted = to_arrow_interval(morsel.column(index))
-                morsel = morsel.set_column(index, column.schema_column.identity, converted)
+            column_array = morsel.column(index)
+            column_identity = column.schema_column.identity
+            column_type = column.schema_column.type
+
+            if column_type == OrsoTypes.INTERVAL:
+                converted = to_arrow_interval(column_array)
+                morsel = morsel.set_column(index, column_identity, converted)
+                continue
+
+            if column_type == OrsoTypes.VARCHAR and (
+                pyarrow.types.is_binary(column_array.type)
+                or pyarrow.types.is_large_binary(column_array.type)
+                or pyarrow.types.is_fixed_size_binary(column_array.type)
+            ):
+                converted = column_array.cast(pyarrow.string())
+                morsel = morsel.set_column(index, column_identity, converted)
 
         morsel = morsel.rename_columns(self.final_names)
 

opteryx/planner/logical_planner/logical_planner_builders.py

@@ -199,30 +199,34 @@ def cast(branch, alias: Optional[List[str]] = None, key=None):
         data_type = type_key
     if "Custom" in data_type:
         data_type = branch["data_type"]["Custom"][0][0]["Identifier"]["value"].upper()
-    if data_type == "Timestamp":
+    lower_data_type = data_type.lower()
+    upper_data_type = data_type.upper()
+    if lower_data_type == "timestamp":
         data_type = "TIMESTAMP"
-    elif data_type == "Date":
+    elif lower_data_type == "date":
         data_type = "DATE"
-    elif "Varchar" in data_type:
+    elif "varchar" in lower_data_type:
         data_type = "VARCHAR"
-    elif "Decimal" in data_type:
+    elif "decimal" in lower_data_type:
         data_type = "DECIMAL"
         if "PrecisionAndScale" in branch["data_type"]["Decimal"]:
             precision = branch["data_type"]["Decimal"]["PrecisionAndScale"][0]
             scale = branch["data_type"]["Decimal"]["PrecisionAndScale"][1]
             args.append(build_literal_node(precision))
             args.append(build_literal_node(scale))
-    elif "Integer" in data_type:
+    elif "integer" in lower_data_type:
         data_type = "INTEGER"
-    elif "Double" in data_type:
+    elif "double" in lower_data_type:
         data_type = "DOUBLE"
-    elif "Boolean" in data_type:
+    elif "boolean" in lower_data_type:
         data_type = "BOOLEAN"
-    elif "STRUCT" in data_type:
+    elif "struct" in lower_data_type:
         data_type = "STRUCT"
-    elif "Blob" in data_type:
+    elif "blob" in lower_data_type:
         data_type = "BLOB"
-    elif "Array" in data_type:
+    elif any(token in lower_data_type for token in ("varbinary", "binary", "raw")):
+        data_type = "VARBINARY"
+    elif "array" in lower_data_type:
         element_key = branch["data_type"]["Array"].get("AngleBracket", {"Varchar": None})
         if isinstance(element_key, dict):
             element_key = next(iter(element_key))
@@ -231,23 +235,19 @@ def cast(branch, alias: Optional[List[str]] = None, key=None):
             args.append(element_key)
         data_type = "ARRAY"
     else:
-        if data_type in ("String", "Char", "Text", "Nvarchar"):
+        if upper_data_type in ("STRING", "CHAR", "TEXT", "NVARCHAR"):
             raise SqlError(
                 f"Unsupported type for CAST - '{data_type.upper()}'. Did you mean 'VARCHAR'?"
             )
-        if data_type in ("Float", "Numeric", "Real"):
+        if upper_data_type in ("FLOAT", "NUMERIC", "REAL"):
             raise SqlError(
                 f"Unsupported type for CAST - '{data_type.upper()}'. Did you mean 'DOUBLE'?"
             )
-        if data_type in ("Binary", "Raw", "VarBinary"):
-            raise SqlError(
-                f"Unsupported type for CAST - '{data_type.upper()}'. Did you mean 'BLOB'?"
-            )
-        if data_type in ("Int", "SmallInt", "TinyInt", "BigInt", "BYTE"):
+        if upper_data_type in ("INT", "SMALLINT", "TINYINT", "BIGINT", "BYTE"):
             raise SqlError(
                 f"Unsupported type for CAST - '{data_type.upper()}'. Did you mean 'INTEGER'?"
             )
-        if data_type in ("Bool", "Bit"):
+        if upper_data_type in ("BOOL", "BIT"):
             raise SqlError(
                 f"Unsupported type for CAST - '{data_type.upper()}'. Did you mean 'BOOLEAN'?"
             )

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "opteryx"
-version = "0.26.2-beta.1894"
+version = "0.26.2-beta.1895"
 description = "Query your data, where it lives"
 requires-python = '>=3.11'
 readme = {file = "README.md", content-type = "text/markdown"}

tests/integration/sql_battery/test_casts_battery.py

@@ -97,9 +97,14 @@
     ("SELECT CAST(CAST(CAST('456' AS INTEGER) AS DOUBLE) AS VARCHAR)", 1, 1, None),
     ("SELECT CAST(CAST(TRUE AS INTEGER) AS VARCHAR)", 1, 1, None),
 
-    # BLOB/BINARY casts
+    # VARBINARY casts
     ("SELECT CAST('test' AS BLOB)", 1, 1, None),
     ("SELECT CAST(CAST('test' AS BLOB) AS VARCHAR)", 1, 1, None),
+    ("SELECT CAST('test' AS VARBINARY)", 1, 1, None),
+    ("SELECT VARBINARY('test')", 1, 1, None),
+    ("SELECT TRY_VARBINARY('test')", 1, 1, None),
+    ("SELECT 'test'::VARBINARY", 1, 1, None),
+    ("SELECT CAST(CAST('test' AS VARBINARY) AS VARCHAR)", 1, 1, None),
 
 ]
 # fmt:on