Improve version handling (#317)

Author: cthoyt

pyproject.toml

@@ -70,7 +70,7 @@ dependencies = [
     "tabulate",
     "cachier",
     "pystow>=0.7.0",
-    "bioversions>=0.8.44",
+    "bioversions>=0.8.101",
     "bioregistry>=0.12.30",
     "bioontologies>=0.7.2",
     "ssslm>=0.0.13",

src/pyobo/constants.py

@@ -4,6 +4,8 @@
 
 import logging
 import re
+from pathlib import Path
+from typing import Literal, NamedTuple, TypeAlias
 
 import pystow
 from typing_extensions import NotRequired, TypedDict
@@ -188,6 +190,8 @@ class IterHelperHelperDict(SlimGetOntologyKwargs):
     skip_set: set[str] | None
 
 
+OntologyFormat: TypeAlias = Literal["obo", "owl", "json", "rdf"]
+
 #: from table 2 of the Functional OWL syntax definition
 #: at https://www.w3.org/TR/owl2-syntax/#IRIs
 DEFAULT_PREFIX_MAP = {
@@ -196,3 +200,10 @@ class IterHelperHelperDict(SlimGetOntologyKwargs):
     "xsd": "http://www.w3.org/2001/XMLSchema#",
     "owl": "http://www.w3.org/2002/07/owl#",
 }
+
+
+class OntologyPathPack(NamedTuple):
+    """A format and path tuple."""
+
+    format: OntologyFormat
+    path: Path

src/pyobo/getters.py

@@ -15,7 +15,7 @@
 from collections.abc import Callable, Iterable, Mapping, Sequence
 from pathlib import Path
 from textwrap import indent
-from typing import Any, Literal, TypeAlias, TypeVar
+from typing import Any, TypeVar
 
 import bioontologies.robot
 import bioregistry
@@ -30,19 +30,21 @@
     DATABASE_DIRECTORY,
     GetOntologyKwargs,
     IterHelperHelperDict,
+    OntologyFormat,
+    OntologyPathPack,
     SlimGetOntologyKwargs,
 )
 from .identifier_utils import ParseError, wrap_norm_prefix
 from .plugins import has_nomenclature_plugin, run_nomenclature_plugin
 from .struct import Obo
 from .struct.obo import from_obo_path, from_obonet
 from .utils.io import safe_open_writer
+from .utils.misc import VERSION_GETTERS, cleanup_version
 from .utils.path import ensure_path, prefix_directory_join
 from .version import get_git_hash, get_version
 
 __all__ = [
     "NoBuildError",
-    "OntologyFormat",
     "get_ontology",
 ]
 
@@ -125,10 +127,20 @@ def get_ontology(
     """
     if force:
         force_process = True
+    if has_nomenclature_plugin(prefix):
+        obo = run_nomenclature_plugin(prefix, version=version)
+        if cache:
+            logger.debug("[%s] caching nomenclature plugin", prefix)
+            obo.write_default(force=force_process)
+        return obo
+
     if prefix == "uberon":
         logger.info("UBERON has so much garbage in it that defaulting to non-strict parsing")
         strict = False
 
+    if version is None:
+        version = _get_version_from_artifact(prefix)
+
     if force_process:
         obonet_json_gz_path = None
     elif not cache:
@@ -157,17 +169,11 @@ def get_ontology(
         else:
             logger.debug("[%s] no obonet cache found at %s", prefix, obonet_json_gz_path)
 
-    if has_nomenclature_plugin(prefix):
-        obo = run_nomenclature_plugin(prefix, version=version)
-        if cache:
-            logger.debug("[%s] caching nomenclature plugin", prefix)
-            obo.write_default(force=force_process)
-        return obo
-
-    ontology_format, path = _ensure_ontology_path(prefix, force=force, version=version)
-    if path is None:
+    path_pack = _ensure_ontology_path(prefix, force=force, version=version)
+    if path_pack is None:
         raise NoBuildError(prefix)
-    elif ontology_format == "obo":
+    ontology_format, path = path_pack
+    if ontology_format == "obo":
         pass  # all gucci
     elif ontology_format in {"owl", "rdf"}:
         path = _convert_to_obo(path)
@@ -195,7 +201,6 @@ def get_ontology(
     return obo
 
 
-OntologyFormat: TypeAlias = Literal["obo", "owl", "json", "rdf"]
 # order matters in this list, since order implicitly defines priority
 _ONTOLOGY_GETTERS: list[tuple[OntologyFormat, Callable[[str], str | None]]] = [
     ("obo", bioregistry.get_obo_download),
@@ -206,8 +211,8 @@ def get_ontology(
 
 
 def _ensure_ontology_path(
-    prefix: str, force: bool, version: str | None
-) -> tuple[OntologyFormat, Path] | tuple[None, None]:
+    prefix: str, *, force: bool, version: str | None
+) -> OntologyPathPack | None:
     for ontology_format, getter in _ONTOLOGY_GETTERS:
         url = getter(prefix)
         if url is None:
@@ -219,8 +224,8 @@ def _ensure_ontology_path(
         except pystow.utils.UnexpectedDirectoryError:
             continue  # TODO report more info about the URL and the name it tried to make
         else:
-            return ontology_format, path
-    return None, None
+            return OntologyPathPack(ontology_format, path)
+    return None
 
 
 SKIP = {
@@ -519,3 +524,20 @@ def db_output_helper(
     click.echo()
 
     return [path for _, path in rv]
+
+
+def _get_version_from_artifact(prefix: str) -> str | None:
+    # assume that all possible files that can be downloaded
+    # are in sync and have the same version
+    for ontology_format, func in _ONTOLOGY_GETTERS:
+        url = func(prefix)
+        if url is None:
+            continue
+        # Try to peak into the file to get the version without fully downloading
+        version_func = VERSION_GETTERS.get(ontology_format)
+        if version_func is None:
+            continue
+        version = version_func(prefix, url)
+        if version:
+            return cleanup_version(version, prefix=prefix)
+    return None

src/pyobo/utils/misc.py

@@ -1,9 +1,17 @@
 """Miscellaneous utilities."""
 
+from __future__ import annotations
+
 import logging
+from collections.abc import Callable
 from datetime import datetime
 
+import bioversions.utils
+
+from pyobo.constants import OntologyFormat
+
 __all__ = [
+    "VERSION_GETTERS",
     "cleanup_version",
 ]
 
@@ -43,6 +51,9 @@
 
 def cleanup_version(data_version: str, prefix: str) -> str:
     """Clean the version information."""
+    # in case a literal string that wasn't parsed properly gets put in
+    data_version = data_version.strip('"')
+
     if data_version in VERSION_REWRITES:
         return VERSION_REWRITES[data_version]
 
@@ -79,3 +90,32 @@ def cleanup_version(data_version: str, prefix: str) -> str:
         logger.debug("[%s] bizarre version: %s", prefix, data_version)
         BIZARRE_LOGGED.add((prefix, data_version))
     return data_version
+
+
+def _get_obo_version(prefix: str, url: str, *, max_lines: int = 200) -> str | None:
+    rv = bioversions.utils.get_obo_version(url, max_lines=max_lines)
+    if rv is None:
+        return None
+    return cleanup_version(rv, prefix)
+
+
+def _get_owl_version(prefix: str, url: str, *, max_lines: int = 200) -> str | None:
+    rv = bioversions.utils.get_owl_xml_version(url, max_lines=max_lines)
+    if rv is None:
+        return None
+    return cleanup_version(rv, prefix)
+
+
+def _get_obograph_json_version(prefix: str, url: str) -> str | None:
+    rv = bioversions.utils.get_obograph_json_version(url)
+    if rv is None:
+        return None
+    return cleanup_version(rv, prefix)
+
+
+#: A mapping from data type to gersion getter function
+VERSION_GETTERS: dict[OntologyFormat, Callable[[str, str], str | None]] = {
+    "obo": _get_obo_version,
+    "owl": _get_owl_version,
+    "json": _get_obograph_json_version,
+}