🐛 Fix SQLite IN filters for non-JSON-serializable values (#7486)

Serialize SQLite smart IN-clause values through the column bind
processor before passing them to `json_each`. This supports values
such as datetimes that are not directly JSON serializable, and makes
their JSON representation match how they are stored.

Fixes stfc/aiida-mlip#262, introduced in 8d562b4.

Author: agoscinski

src/aiida/storage/utils.py

@@ -13,10 +13,9 @@
 import json
 from collections.abc import Sequence
 from functools import singledispatch
-from typing import TYPE_CHECKING, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
 
 from sqlalchemy import Select, or_, select, type_coerce
-from sqlalchemy import cast as sql_cast
 from sqlalchemy import func as sa_func
 from sqlalchemy.dialects.postgresql.base import PGDialect
 from sqlalchemy.dialects.sqlite.base import SQLiteDialect
@@ -64,10 +63,18 @@ def _build_select_stmt_psql(dialect: PGDialect, coltype: TypeEngine[T], values:
 
 @_build_select_stmt.register
 def _build_select_stmt_sqlite(dialect: SQLiteDialect, coltype: TypeEngine[T], values: Sequence[T]) -> Select[tuple[T]]:
-    """SQLite: ``SELECT CAST(value AS coltype) FROM json_each(:json)`` — passes the list as 1 parameter."""
-    json_each_table = sa_func.json_each(json.dumps(list(values))).table_valued('value')
-    value_col = json_each_table.c.value
-    return select(sql_cast(expression=value_col, type_=coltype)).select_from(json_each_table)
+    """SQLite: ``SELECT value FROM json_each(:json)`` — passes the list as 1 parameter.
+
+    Values are serialized through the column's bind processor so their JSON representation matches
+    how they are stored; SQLite then applies the column's comparison affinity to evaluate the ``IN``.
+    """
+    processor = coltype.dialect_impl(dialect).bind_processor(dialect)
+
+    def process(value: T) -> Any:
+        return processor(value) if processor is not None and value is not None else value
+
+    json_each_table = sa_func.json_each(json.dumps([process(value) for value in values])).table_valued('value')
+    return select(json_each_table.c.value).select_from(json_each_table)
 
 
 def _create_smarter_in_clause(

tests/orm/test_querybuilder.py

@@ -38,6 +38,17 @@ def test_date_filters_support(self):
         builder = orm.QueryBuilder().append(orm.Node, filters={'ctime': {'>': date.today() - timedelta(days=1)}})
         assert builder.count() == 1
 
+    @pytest.mark.usefixtures('aiida_profile_clean')
+    def test_datetime_in_filter_support(self):
+        """Verify that ``datetime`` values are supported for ``in`` and negated ``in`` filters."""
+        node = orm.Data().store()
+
+        builder = orm.QueryBuilder().append(orm.Data, filters={'ctime': {'in': [node.ctime]}})
+        assert builder.count() == 1
+
+        builder = orm.QueryBuilder().append(orm.Data, filters={'ctime': {'!in': [node.ctime]}})
+        assert builder.count() == 0
+
     def test_ormclass_type_classification(self):
         """This tests the classifications of the QueryBuilder"""
         from aiida.common.exceptions import DbContentError

tests/storage/test_utils.py

@@ -9,6 +9,7 @@
 """Tests for :mod:`aiida.storage.utils`."""
 
 from collections.abc import Generator
+from datetime import datetime, timezone
 from typing import cast
 
 import pytest
@@ -64,6 +65,32 @@ def test_sqlite_batches_large_lists(sqlite_session: SqliteSessionFixture) -> Non
     assert ' OR ' in sql
 
 
+def test_sqlite_in_clause_datetime_values() -> None:
+    """SQLite: datetime values are serialized using the column bind processor."""
+    engine: sa.Engine = sa.create_engine(url='sqlite://')
+    metadata: sa.MetaData = sa.MetaData()
+    table: sa.Table = sa.Table(
+        'items',
+        metadata,
+        sa.Column(name='id', type_=sa.Integer, primary_key=True),
+        sa.Column(name='ctime', type_=sa.DateTime(timezone=True)),
+    )
+    metadata.create_all(bind=engine)
+
+    timestamp = datetime(2026, 7, 23, 12, 0, 0, 123456, tzinfo=timezone.utc)
+    with Session(bind=engine) as session:
+        session.execute(table.insert().values(id=1, ctime=timestamp))
+        session.execute(table.insert().values(id=2, ctime=datetime(2026, 7, 24, tzinfo=timezone.utc)))
+        session.commit()
+
+        in_clause: ColumnElement[bool] = _create_smarter_in_clause(
+            session=session, column=table.c.ctime, values=[timestamp]
+        )
+        result = session.execute(sa.select(table.c.id).where(in_clause)).scalars().all()
+
+    assert result == [1]
+
+
 @pytest.mark.requires_psql
 @pytest.mark.usefixtures('aiida_profile_clean')
 def test_psql_uses_unnest() -> None: