Accommodate ORM schema reorg for search/browse queries (#207)

* Accommodate ORM schema reorg (ord=index, public=payload, derived=computed)

ord-schema's upcoming 0.8 release reorganizes the ORM into role-based schemas:
the served Reaction proto moves to public.reactions, per-dataset metadata
(num_reactions, submitted_at) to public.datasets, and the generated SMILES plus
rdkit_*_id links to the derived.* tables (reaction_smiles, compound_smiles,
product_compound_smiles). The ord.* schema is now a pure search index.

Update the search/browse queries to read from the new locations:
- ReactionSmartsQuery / ReactionComponentQuery join ord.reaction (and the
  compound tables) through the derived SMILES tables to reach the rdkit_*_id
  links instead of the dropped ord.reaction.rdkit_reaction_id /
  compound.rdkit_mol_id columns.
- fetch_reactions reads proto from public.reactions.
- The dataset-stats aggregation joins the derived SMILES tables for the SMILES.
- get_datasets / get_dataset join public.datasets for num_reactions /
  submitted_at.

All joins to the derived tables are INNER, mirroring the prior NULL-column
filtering: update_derived_tables inserts a derived row only when a SMILES can be
generated, so un-derivable entities are dropped exactly as before.

Bump the ord-schema pin to >=0.8,<0.9. uv.lock must be regenerated once 0.8.0
is published.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Accommodate ingest/derive split in setup_test_postgres

ord-schema 0.8 splits dataset loading into ingest (add_dataset) and derivation
(update_derived_data); update the test-postgres helper to call both, mirroring the
ord-schema ORM conftest and preserving the rdkit-cartridge gating.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* Regenerate uv.lock for ord-schema 0.8.0

0.8.0 is now published, so resolve the >=0.8,<0.9 pin against it (drops the
stale 0.7.1 lock) and pull in uuid6, the new transitive dependency.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: skearnes

ord_interface/api/queries.py

@@ -154,7 +154,8 @@ def query_and_parameters(self) -> tuple[str, list]:
         query = """
             SELECT DISTINCT reaction.reaction_id
             FROM ord.reaction
-            JOIN rdkit.reactions ON rdkit.reactions.id = reaction.rdkit_reaction_id
+            JOIN derived.reaction_smiles ON reaction_smiles.reaction_id = reaction.id
+            JOIN rdkit.reactions ON rdkit.reactions.id = reaction_smiles.rdkit_reaction_id
             WHERE rdkit.reactions.reaction @> reaction_from_smarts(%s)
         """
         return query, [self._reaction_smarts]
@@ -323,12 +324,14 @@ def _mols_join(self) -> LiteralString:
             return """
             JOIN ord.reaction_input ON reaction_input.reaction_id = reaction.id
             JOIN ord.compound ON compound.reaction_input_id = reaction_input.id
-            JOIN rdkit.mols ON rdkit.mols.id = compound.rdkit_mol_id
+            JOIN derived.compound_smiles ON compound_smiles.compound_id = compound.id
+            JOIN rdkit.mols ON rdkit.mols.id = compound_smiles.rdkit_mol_id
             """
         return """
             JOIN ord.reaction_outcome ON reaction_outcome.reaction_id = reaction.id
             JOIN ord.product_compound ON product_compound.reaction_outcome_id = reaction_outcome.id
-            JOIN rdkit.mols ON rdkit.mols.id = product_compound.rdkit_mol_id
+            JOIN derived.product_compound_smiles ON product_compound_smiles.product_compound_id = product_compound.id
+            JOIN rdkit.mols ON rdkit.mols.id = product_compound_smiles.rdkit_mol_id
             """
 
     @property
@@ -544,9 +547,10 @@ async def fetch_reactions(
     """
     unique_ids = list(dict.fromkeys(reaction_ids))  # De-dup, preserving order.
     query = """
-        SELECT dataset.dataset_id, reaction.reaction_id, reaction.proto
+        SELECT dataset.dataset_id, reaction.reaction_id, reactions.proto
         FROM ord.reaction
         JOIN ord.dataset ON dataset.id = reaction.dataset_id
+        JOIN public.reactions ON reactions.reaction_id = reaction.reaction_id
         WHERE reaction.reaction_id = ANY (%s)
     """
     await cursor.execute(query, (unique_ids,))
@@ -574,39 +578,54 @@ async def _fetch_dataset_most_used_smiles(
     compound_table: str,
     join_table: str,
     foreign_key: str,
+    smiles_table: str,
+    smiles_foreign_key: str,
     limit: int,
 ) -> list[StatsResult]:
     """Fetches the top K most used SMILES for a dataset, joined through the given tables.
 
     Args:
         cursor: Database cursor.
         dataset_id: Dataset to aggregate over.
-        compound_table: Table holding SMILES, e.g. "compound" or "product_compound".
+        compound_table: Compound table linking to reactions, e.g. "compound" or
+            "product_compound".
         join_table: Table linking compounds to reactions, e.g. "reaction_input".
         foreign_key: Column on ``compound_table`` referencing ``join_table``.
+        smiles_table: Derived table holding the generated SMILES, e.g.
+            "compound_smiles" or "product_compound_smiles".
+        smiles_foreign_key: Column on ``smiles_table`` referencing ``compound_table``.
         limit: Maximum number of rows to return.
 
     Returns:
         Most frequently appearing SMILES, in descending order of frequency.
     """
     # Compose schema-qualified identifiers safely rather than interpolating raw
-    # strings into the SQL text.
+    # strings into the SQL text. The SMILES live in the derived schema, one row per
+    # compound, joined back to the compound table by its primary key.
     compound = sql.Identifier("ord", compound_table)
     join = sql.Identifier("ord", join_table)
+    smiles = sql.Identifier("derived", smiles_table)
     query = sql.SQL(
         """
-            SELECT smiles, COUNT(*) AS times_appearing
+            SELECT {smiles}.smiles AS smiles, COUNT(*) AS times_appearing
             FROM {compound}
             JOIN {join} ON {compound}.{foreign_key} = {join}.id
             JOIN ord.reaction ON {join}.reaction_id = ord.reaction.id
             JOIN ord.dataset ON ord.reaction.dataset_id = ord.dataset.id
+            JOIN {smiles} ON {smiles}.{smiles_foreign_key} = {compound}.id
             WHERE ord.dataset.dataset_id = %s
-            AND smiles IS NOT NULL
-            GROUP BY smiles
+            AND {smiles}.smiles IS NOT NULL
+            GROUP BY {smiles}.smiles
             ORDER BY times_appearing DESC
             LIMIT %s
         """
-    ).format(compound=compound, join=join, foreign_key=sql.Identifier(foreign_key))
+    ).format(
+        compound=compound,
+        join=join,
+        foreign_key=sql.Identifier(foreign_key),
+        smiles=smiles,
+        smiles_foreign_key=sql.Identifier(smiles_foreign_key),
+    )
     await cursor.execute(query, (dataset_id, limit))
     results = []
     async for row in cursor:
@@ -624,6 +643,8 @@ async def fetch_dataset_most_used_smiles_for_inputs(
         compound_table="compound",
         join_table="reaction_input",
         foreign_key="reaction_input_id",
+        smiles_table="compound_smiles",
+        smiles_foreign_key="compound_id",
         limit=limit,
     )
 
@@ -638,5 +659,7 @@ async def fetch_dataset_most_used_smiles_for_products(
         compound_table="product_compound",
         join_table="reaction_outcome",
         foreign_key="reaction_outcome_id",
+        smiles_table="product_compound_smiles",
+        smiles_foreign_key="product_compound_id",
         limit=limit,
     )

ord_interface/api/queries_test.py

@@ -185,13 +185,15 @@ async def _max_similarities(test_cursor, reaction_ids, pattern, target):
         join = """
         JOIN ord.reaction_input ON reaction_input.reaction_id = reaction.id
         JOIN ord.compound ON compound.reaction_input_id = reaction_input.id
-        JOIN rdkit.mols ON rdkit.mols.id = compound.rdkit_mol_id
+        JOIN derived.compound_smiles ON compound_smiles.compound_id = compound.id
+        JOIN rdkit.mols ON rdkit.mols.id = compound_smiles.rdkit_mol_id
         """
     else:
         join = """
         JOIN ord.reaction_outcome ON reaction_outcome.reaction_id = reaction.id
         JOIN ord.product_compound ON product_compound.reaction_outcome_id = reaction_outcome.id
-        JOIN rdkit.mols ON rdkit.mols.id = product_compound.rdkit_mol_id
+        JOIN derived.product_compound_smiles ON product_compound_smiles.product_compound_id = product_compound.id
+        JOIN rdkit.mols ON rdkit.mols.id = product_compound_smiles.rdkit_mol_id
         """
     await test_cursor.execute(
         f"""

ord_interface/api/search.py

@@ -263,8 +263,11 @@ async def get_datasets() -> list[DatasetInfo]:
     """Returns info about the current datasets, most recently submitted first."""
     async with get_cursor() as cursor:
         await cursor.execute(
-            "SELECT dataset_id, name, description, num_reactions, submitted_at FROM dataset "
-            "ORDER BY submitted_at DESC NULLS LAST, dataset_id"
+            "SELECT dataset.dataset_id, dataset.name, dataset.description, "
+            "datasets.num_reactions, datasets.submitted_at "
+            "FROM ord.dataset "
+            "JOIN public.datasets ON datasets.dataset_id = dataset.dataset_id "
+            "ORDER BY datasets.submitted_at DESC NULLS LAST, dataset.dataset_id"
         )
         return [DatasetInfo(**row) async for row in cursor]
 
@@ -274,7 +277,11 @@ async def get_dataset(dataset_id: str) -> DatasetInfo:
     """Returns info about a single dataset."""
     async with get_cursor() as cursor:
         await cursor.execute(
-            "SELECT dataset_id, name, description, num_reactions, submitted_at FROM dataset WHERE dataset_id = %s",
+            "SELECT dataset.dataset_id, dataset.name, dataset.description, "
+            "datasets.num_reactions, datasets.submitted_at "
+            "FROM ord.dataset "
+            "JOIN public.datasets ON datasets.dataset_id = dataset.dataset_id "
+            "WHERE dataset.dataset_id = %s",
             (dataset_id,),
         )
         row = await cursor.fetchone()

ord_interface/api/testing.py

@@ -18,7 +18,7 @@
 from glob import glob
 
 from ord_schema.message_helpers import load_message
-from ord_schema.orm.database import add_dataset, prepare_database
+from ord_schema.orm.database import add_dataset, prepare_database, update_derived_data
 from ord_schema.proto import dataset_pb2
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
@@ -38,4 +38,7 @@ def setup_test_postgres(url: str) -> None:
     with Session(engine) as session:
         for dataset in datasets:
             with session.begin():
-                add_dataset(dataset, session, rdkit_cartridge=rdkit_cartridge)
+                add_dataset(dataset, session)
+                update_derived_data(
+                    dataset.dataset_id, session, rdkit_cartridge=rdkit_cartridge
+                )

pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "gunicorn",
     "jinja2>=2.0.0",
     "numpy>=1.26.4",
-    "ord-schema[orm]>=0.7.1,<0.8",
+    "ord-schema[orm]>=0.8,<0.9",
     "pandas>=1.0.4",
     "protobuf>=4.22,<6",
     "psycopg[binary,pool]>=3",