Merge pull request #34 from BioImage-Archive/use_ontology_labels

Use ontology labels

Author: sherwoodf

.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      elasticsearch:
+        image: elasticsearch:8.11.0
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+          ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+        ports:
+          - 9200:9200
+        options: >-
+          --health-cmd "curl -f http://localhost:9200/_cluster/health || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - name: Install dependencies
+      run: uv sync --dev
+
+    - name: Wait for Elasticsearch
+      run: |
+        until curl -f http://localhost:9200/_cluster/health; do
+          echo "Waiting for Elasticsearch..."
+          sleep 5
+        done
+
+    - name: Run tests
+      run: uv run pytest tests/ -v

src/gide_search/search/schema_search_object.py

@@ -1,15 +1,31 @@
+import logging
+
+from typing import Protocol
 from urllib import parse
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationInfo,
+    field_validator,
+    model_validator,
+)
 from typing_extensions import Self
 
+logger = logging.getLogger()
+
 # Prefixes that might occur in object IDs that are likely to get shortened, which would be better left as full IRIs.
 PREFIXES_TO_EXPAND = {
     "obo": "http://purl.obolibrary.org/obo/",
     "bao": "http://www.bioassayontology.org/bao#",
 }
 
 
+class TermLabelProvider(Protocol):
+    def fetch_label_by_iri(self, term_iri: str) -> str | None: ...
+
+
 class JsonLdNode(BaseModel):
     model_config = ConfigDict(
         populate_by_name=True,  # accept `id` AND `@id`
@@ -152,7 +168,7 @@ class Dataset(JsonLdNode):
 
     @field_validator("about", mode="before")
     @classmethod
-    def discriminate_about(cls, value):
+    def discriminate_about(cls, value, info: ValidationInfo):
         if not isinstance(value, list):
             return value
         out = []
@@ -175,7 +191,7 @@ def discriminate_about(cls, value):
 
     @field_validator("measurementMethod", mode="before")
     @classmethod
-    def discriminate_measurement_method(cls, value):
+    def discriminate_measurement_method(cls, value, info: ValidationInfo):
         if not isinstance(value, list):
             return value
         out = []
@@ -210,7 +226,7 @@ class IndexableDataset(Dataset):
     imaging_method_ids: list[DefinedTerm] = Field(default_factory=list)
 
     @model_validator(mode="after")
-    def poplate_additional_index_fields(self) -> Self:
+    def populate_additional_index_fields(self) -> Self:
         """
         Populate the fields that get used for facetting
         """
@@ -224,3 +240,14 @@ def poplate_additional_index_fields(self) -> Self:
             ):
                 self.imaging_method_ids.append(measurment_object)
         return self
+
+    def fetch_labels(self, label_provider: TermLabelProvider) -> None:
+        def _fetch_for_defined_term(term: DefinedTerm):
+            label = label_provider.fetch_label_by_iri(term.id)
+            if label:
+                term.name = label
+            else:
+                logger.warning(f"{term.id} not found in ontology.")
+
+        for term in self.imaging_method_ids:
+            _fetch_for_defined_term(term)

src/gide_search/transformers/bia_to_rocrate.py

@@ -125,7 +125,7 @@ def _get_taxons_from_ontology(self, bia_bio_sample):
                     else:
                         continue
 
-                term_with_labels = self.ontology_term_finder.fetch_labels_for_term(
+                term_with_labels = self.ontology_term_finder.fetch_term_from_ontology(
                     "ncbitaxon", ncbi_id
                 )
                 if term_with_labels:
@@ -217,7 +217,7 @@ def _get_imaging_method_from_ontology(self, bia_image_acquisition_protocol):
                     )
         else:
             for fbbi_id in bia_image_acquisition_protocol["fbbi_id"]:
-                term_with_labels = self.ontology_term_finder.fetch_labels_for_term(
+                term_with_labels = self.ontology_term_finder.fetch_term_from_ontology(
                     "fbbi", fbbi_id
                 )
                 if term_with_labels:

src/gide_search/transformers/frame_transformer.py

@@ -38,6 +38,6 @@ def _get_ro_crate_context_with_containers(self) -> list | str:
                     "@id": "http://schema.org/taxonomicRange",
                     "@container": "@set",
                 },
-                "@type": {"@container": "@set"},
+                "@type": {"@container": "@set"}
             },
         ]

src/gide_search/transformers/rocrate_to_index.py

@@ -5,6 +5,7 @@
 
 from gide_search.search.schema_search_object import IndexableDataset
 from gide_search.transformers.frame_transformer import FrameTransformer
+from gide_search.utils.ontology_term_finder import OntologyTermFinder
 
 logger = logging.getLogger("__main__." + __name__)
 
@@ -21,16 +22,23 @@ def __init__(self):
         self.frame = self.FRAME_BASE | {
             "@context": self._get_ro_crate_context_with_containers()
         }
+        self.ontology_lookup = OntologyTermFinder()
         super().__init__()
 
     def transform(self, single_object: dict):
 
-        # FIXME: currently replacing context with defined one while we all update our ro-crates.
-        single_object["@context"] = (
-            "https://www.gide-project.org/ro-crate/search/1.0/context"
-        )
+        base_iri = self._find_root_object(single_object).get("about", {}).get("@id")
+
+        # FIXME: currently replacing context with defined one while we all update our ro-crates. The base IRI still needs to be present.
+        single_object["@context"] = [
+            "https://www.gide-project.org/ro-crate/search/1.0/context",
+            {"@base": base_iri},
+        ]
 
-        framed_doc = jsonld.frame(single_object, self.frame)
+        framed_doc = jsonld.frame(
+            single_object,
+            self.frame,
+        )
 
         if not isinstance(framed_doc, dict):
             raise TypeError()
@@ -39,10 +47,26 @@ def transform(self, single_object: dict):
 
         try:
             dataset = IndexableDataset.model_validate(framed_doc)
+            dataset.fetch_labels(self.ontology_lookup)
         except ValidationError as e:
             logger.error(
-                f"Validation failed for: {framed_doc.get("@id", "Unknown object")}"
+                f"Validation failed for: {framed_doc.get('@id', 'Unknown object')}"
             )
             raise e
 
         return dataset.model_dump(by_alias=False)
+
+    @staticmethod
+    def _find_root_object(ro_crate_metadata: dict) -> dict:
+        for entity in ro_crate_metadata.get("@graph"):
+            if entity.get("@id") == "ro-crate-metadata.json":
+                return entity
+
+        # Waiting on update to example ro-crate to correct mistake we made about the ID of the root object.
+        for entity in ro_crate_metadata.get("@graph"):
+            if entity.get("@type") == "CreativeWork" and all(
+                (x in entity for x in ("about", "conformsTo"))
+            ):
+                return entity
+
+        raise ValueError("Cannot find root entity in ro-crate document")

src/gide_search/utils/ontology_term_finder.py

@@ -10,7 +10,7 @@
 
 
 @dataclass
-class TermWithLabels:
+class OntologyTerm:
     iri: str
     label: list[str]
     additional_label: list[str]
@@ -58,7 +58,9 @@ def _collect_short_ids(short_id: str | list[str], short_ids: list):
                 short_ids += short_id
 
     @cache
-    def fetch_labels_for_term(self, ontology: str, term_iri: str):
+    def fetch_term_from_ontology(
+        self, ontology: str, term_iri: str
+    ) -> None | OntologyTerm:
         if ontology not in self.avaliable_ontology_ids:
             raise KeyError(f"{ontology} is not in ols")
 
@@ -75,13 +77,45 @@ def fetch_labels_for_term(self, ontology: str, term_iri: str):
 
         return self._create_term_with_labels(term_info)
 
+    def fetch_term_by_iri(self, term_iri: str) -> None | OntologyTerm:
+        ontology = self._ontology_for_term_iri(term_iri)
+        if ontology is None:
+            return
+
+        try:
+            return self.fetch_term_from_ontology(ontology, term_iri)
+        except KeyError:
+            return
+
+    def fetch_label_by_iri(self, term_iri: str) -> str | None:
+        term_with_labels = self.fetch_term_by_iri(term_iri)
+        if term_with_labels is None:
+            return None
+        return term_with_labels.label[0] if term_with_labels.label else None
+
+    def _ontology_for_term_iri(self, term_iri: str) -> str | None:
+        if term_iri.startswith("obo:"):
+            term_iri = term_iri.removeprefix("obo:")
+
+        if term_iri.startswith("http://purl.obolibrary.org/obo/"):
+            local_part = term_iri.rsplit("/", 1)[-1]
+            prefix = local_part.split("_", 1)[0].lower()
+            return prefix
+
+        if term_iri.startswith("http://www.bioassayontology.org/bao#"):
+            return "bao"
+        if term_iri.startswith("bao:"):
+            return "bao"
+
+        return None
+
     @cache
     def _get_iri_for_class_in_ontology(
         self, ontology: str, search_terms: str, required_superclass: str | None = None
-    ) -> list[TermWithLabels]:
+    ) -> list[OntologyTerm]:
         api_response = self._find_class_in_ontology(ontology, search_terms)
 
-        iris_and_labels: list[TermWithLabels] = []
+        iris_and_labels: list[OntologyTerm] = []
         for ontology_term in api_response["elements"]:
             if required_superclass:
                 if not ontology_term["hasDirectParents"]:
@@ -97,7 +131,7 @@ def _get_iri_for_class_in_ontology(
 
         return iris_and_labels
 
-    def _create_term_with_labels(self, ontology_term) -> TermWithLabels:
+    def _create_term_with_labels(self, ontology_term) -> OntologyTerm:
         short_ids = []
         self._collect_short_ids(ontology_term.get("obo_id"), short_ids)
         self._collect_short_ids(ontology_term.get("curie"), short_ids)
@@ -109,7 +143,7 @@ def _create_term_with_labels(self, ontology_term) -> TermWithLabels:
             else [ontology_term["label"]]
         )
 
-        return TermWithLabels(
+        return OntologyTerm(
             **{
                 "iri": ontology_term["iri"],
                 "label": label,

tests/conftest.py

@@ -0,0 +1,60 @@
+"""Shared pytest fixtures for tests."""
+
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from gide_search.cli import app
+from gide_search.search.indexer import DatabaseEntryIndexer
+
+runner = CliRunner()
+
+
+def is_elasticsearch_available(es_url: str = "http://localhost:9200") -> bool:
+    """Check if Elasticsearch is available."""
+    try:
+        indexer = DatabaseEntryIndexer(es_url=es_url)
+        return indexer.ping()
+    except Exception:
+        return False
+
+
+@pytest.fixture
+def es_available():
+    """Skip test if Elasticsearch is not available."""
+    if not is_elasticsearch_available():
+        pytest.skip("Elasticsearch is not available on localhost:9200")
+
+
+@pytest.fixture
+def indexed_data(es_available):
+    """Fixture to ensure sample data is indexed before tests."""
+    sample_index_file = (
+        Path(__file__).parent
+        / "data"
+        / "index_document"
+        / "example_ro_crate_index.json"
+    )
+
+    if not sample_index_file.exists():
+        pytest.skip(f"Sample index file not found: {sample_index_file}")
+
+    # Run the index command
+    result = runner.invoke(
+        app,
+        [
+            "data",
+            "index",
+            str(sample_index_file),
+            "--es-url",
+            "http://localhost:9200",
+            "--recreate",
+        ],
+    )
+
+    if result.exit_code != 0:
+        pytest.skip(f"Failed to index data: {result.stdout}")
+
+    # Return something to indicate success
+    return True

tests/data/gide_search_ro_crate/EXAMPLE-001-ro-crate-metadata.json

@@ -201,7 +201,7 @@
         {
             "@id": "http://purl.obolibrary.org/obo/CLO_0003684",
             "@type": "DefinedTerm",
-            "name": "Hela cell"
+            "name": "HeLa cell"
         },
         {
             "@id": "#total-dataset-size",