Implement standard SSSOM hashing function

pyproject.toml

@@ -141,7 +141,7 @@ fastapi = [
 ]
 wikidata = [
     "wikidata-client",
-    "quickstatements_client",
+    "quickstatements-client>=0.1.2",
     "bioregistry",
 ]
 

src/sssom_pydantic/api.py

@@ -4,7 +4,6 @@
 
 import datetime
 import functools
-import hashlib
 from collections.abc import Callable
 from typing import Annotated, Any, Literal, TypeAlias
 
@@ -22,7 +21,7 @@
     EntityTypeLiteral,
     Row,
 )
-from .models import Cardinality, Record
+from .models import Cardinality, Record, expanded_record_to_str
 
 __all__ = [
     "NOT",
@@ -35,7 +34,7 @@
     "SemanticMapping",
     "SemanticMappingHash",
     "SemanticMappingPredicate",
-    "mapping_hash_v1",
+    "hash_mapping_to_reference",
 ]
 
 PredicateModifier: TypeAlias = Literal["Not"]
@@ -691,21 +690,45 @@ def to_record(self) -> ExtensionDefinitionRecord:
         )
 
 
-MAPPING_HASH_V1_PREFIX = "sssom-pydantic-mapping-hash-v2"
-MAPPING_HASH_V1_EXCLUDE: set[str] = {"record", "cardinality"}
-MAPPING_HASH_V1_URI_PREFIX = f"https://w3id.org/sssom/{MAPPING_HASH_V1_PREFIX}/"
+MAPPING_HASH_CURIE_PREFIX = "sssom.record"
+MAPPING_HASH_URI_PREFIX = "https://w3id.org/sssom/record/"
 
 
-def mapping_hash_v1(m: SemanticMapping, converter: curies.Converter) -> Reference:
+def hash_mapping_to_reference(mapping: SemanticMapping, converter: curies.Converter) -> Reference:
     """Hash a mapping into a reference."""
-    h = hashlib.md5(usedforsecurity=False)
-    h.update(
-        m.model_dump_json(
-            exclude=MAPPING_HASH_V1_EXCLUDE,
-            exclude_none=True,
-            exclude_unset=True,
-            exclude_defaults=True,
-        ).encode("utf8")
-    )
-    identifier = h.hexdigest()
-    return Reference(prefix=MAPPING_HASH_V1_PREFIX, identifier=identifier)
+    identifier = hash_mapping(mapping, converter)
+    return Reference(prefix=MAPPING_HASH_CURIE_PREFIX, identifier=identifier)
+
+
+def hash_mapping(mapping: SemanticMapping, converter: curies.Converter) -> str:
+    """Hash the mapping.
+
+    :param mapping: A semantic mapping
+    :param converter: A converter
+
+    :returns: A hexadecimal representation of the FNV64 hash of the canonical
+        S-expression for the mapping, proposed in
+        https://github.com/mapping-commons/sssom/pull/534.
+    """
+    return _fnv64(mapping_to_sexpr_str(mapping, converter).encode("utf-8")).hex().upper()
+
+
+FNV64_PRIME = 1099511628211
+FNV64_OFFSET = 14695981039346656037
+FNV64_MOD = 2**64
+
+
+def _fnv64(data: bytes) -> bytes:
+    h = FNV64_OFFSET
+    for byte in data:
+        h ^= byte
+        h = (h * FNV64_PRIME) % FNV64_MOD
+    return h.to_bytes(8, "little")
+
+
+def mapping_to_sexpr_str(
+    mapping: SemanticMapping, converter: curies.Converter, *, _debug: bool = False
+) -> str:
+    """Convert a mapping to a S-expression string."""
+    expanded_record = mapping.to_record().expand(converter)
+    return expanded_record_to_str(expanded_record, _debug=_debug)

src/sssom_pydantic/database/repo.py

@@ -16,7 +16,7 @@
     MappingSetRecord,
     SemanticMapping,
     SemanticMappingHash,
-    mapping_hash_v1,
+    hash_mapping_to_reference,
 )
 from ..io import Metadata, read, write
 from ..process import Mark, curate, estimate_confidence, publish, review
@@ -39,7 +39,7 @@ def __init__(
         converter: Converter,
     ) -> None:
         """Initialize the repository."""
-        self.semantic_mapping_hash = semantic_mapping_hash or mapping_hash_v1
+        self.semantic_mapping_hash = semantic_mapping_hash or hash_mapping_to_reference
         self.converter = converter
 
     def hash_mapping(self, mapping: SemanticMapping) -> Reference:

src/sssom_pydantic/examples.py

@@ -14,11 +14,11 @@
 from pydantic import BaseModel
 
 from sssom_pydantic.api import (
-    MAPPING_HASH_V1_PREFIX,
-    MAPPING_HASH_V1_URI_PREFIX,
+    MAPPING_HASH_CURIE_PREFIX,
+    MAPPING_HASH_URI_PREFIX,
     MappingTool,
     SemanticMapping,
-    mapping_hash_v1,
+    hash_mapping_to_reference,
 )
 
 __all__ = [
@@ -50,7 +50,7 @@
     #
     "spdx": "https://spdx.org/licenses/",
     "w3id": "https://w3id.org/",
-    MAPPING_HASH_V1_PREFIX: MAPPING_HASH_V1_URI_PREFIX,
+    MAPPING_HASH_CURIE_PREFIX: MAPPING_HASH_URI_PREFIX,
     "issue": "https://github.com/cthoyt/sssom-pydantic/issues/",
     "biolink": "https://w3id.org/biolink/vocab/",
     "rule": "https://example.org/disease-rule/",
@@ -90,7 +90,9 @@ class ExampleMapping(BaseModel):
 
 e1_with_hash = ExampleMapping(
     description="reference for the mapping itself in the `record` field",
-    semantic_mapping=simple.model_copy(update={"record": mapping_hash_v1(simple, TEST_CONVERTER)}),
+    semantic_mapping=simple.model_copy(
+        update={"record": hash_mapping_to_reference(simple, TEST_CONVERTER)}
+    ),
 )
 
 simple_with_author = ExampleMapping(

src/sssom_pydantic/models.py

@@ -3,8 +3,8 @@
 from __future__ import annotations
 
 import datetime
-from collections.abc import Callable
-from typing import TYPE_CHECKING, Literal, TypeAlias
+from collections.abc import Callable, Sequence
+from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias
 
 from curies.vocabulary import matching_processes
 from pydantic import AnyUrl, BaseModel, ConfigDict, Field
@@ -235,3 +235,81 @@ def compress(self, converter: curies.Converter) -> Record:
             if uris := data.get(key):
                 data[key] = [converter.compress(str(uri), strict=True) for uri in uris]
         return Record.model_validate(data)
+
+
+SKIP_SLOTS = {"record_id", "mapping_cardinality"}
+
+
+def expanded_record_to_str(mapping: ExpandedRecord, *, _debug: bool = False) -> str:
+    """Convert a mapping to a S-expression string."""
+    return box_to_str(expanded_record_to_box(mapping), _debug=_debug)
+
+
+def expanded_record_to_box(record: ExpandedRecord) -> Box:
+    """Convert a mapping to a S-expression."""
+    boxes: list[Box] = []
+    for name in ExpandedRecord.model_fields:
+        if name in SKIP_SLOTS:
+            continue
+        match getattr(record, name, None):
+            case None:
+                continue
+            case str() | float() | bool() | datetime.date() as value:
+                boxes.append(Box(name, value))
+            case AnyUrl() as url:
+                boxes.append(Box(name, str(url)))
+            case list() as values:
+                if not values:
+                    continue
+                if all(isinstance(v, str) for v in values):
+                    boxes.append(Box(name, values))
+                elif all(isinstance(v, AnyUrl) for v in values):
+                    boxes.append(Box(name, [str(v) for v in values]))
+                else:
+                    raise TypeError(f"invalid box value: {values}")
+            case _ as value:
+                raise NotImplementedError(f"not implemented for {type(value)}")
+    return Box("mapping", boxes)
+
+
+class Box(NamedTuple):
+    """A value."""
+
+    label: str
+    value: str | float | bool | datetime.date | Sequence[str | float | bool | datetime.date | Box]
+
+
+def box_to_str(box: Box, *, max_precision: int = 4, _debug: bool = False) -> str:
+    """Convert a S-expression object to a string."""
+    start = f"{len(box.label)}:{box.label}"
+    match box.value:
+        case str() | float() | bool() | datetime.date():
+            return f"({start}{_fmt_primitive(box.value, max_precision=max_precision)})"
+        case list():
+            rr = []
+            for value in box.value:
+                match value:
+                    case str() | float() | bool():
+                        rr.append(_fmt_primitive(value, max_precision=max_precision))
+                    case Box():
+                        rr.append(box_to_str(value, max_precision=max_precision, _debug=_debug))
+            if _debug:
+                inside = "\n".join(rr)
+            else:
+                inside = "".join(rr)
+            return f"({start}({inside}))"
+        case _:
+            raise TypeError(f"invalid box value: {box.value}")
+
+
+def _fmt_primitive(value: str | float | bool | datetime.date, *, max_precision: int = 4) -> str:
+    match value:
+        case str():
+            pass
+        case float():
+            value = str(round(value, max_precision))
+        case bool():
+            raise NotImplementedError
+        case datetime.date():
+            value = value.strftime("%Y-%m-%d")
+    return f"{len(value)}:{value}"

tests/cases.py

@@ -22,7 +22,7 @@
 
 import sssom_pydantic
 from sssom_pydantic import MappingSetRecord
-from sssom_pydantic.api import MAPPING_HASH_V1_PREFIX, SemanticMapping
+from sssom_pydantic.api import MAPPING_HASH_CURIE_PREFIX, SemanticMapping, mapping_to_sexpr_str
 from sssom_pydantic.database import (
     NEGATIVE_MAPPING_CLAUSE,
     POSITIVE_MAPPING_CLAUSE,
@@ -117,12 +117,17 @@ def assert_model_equal(
         expected: SemanticMapping,
         actual: SemanticMapping | None,
         msg: str | None = None,
-        *,
-        skip_name_check: bool | None = None,
     ) -> None:
         """Assert two models are equal."""
         if actual is None:
             raise self.fail()
+
+        if hasattr(self, "repository"):
+            self.assertEqual(
+                mapping_to_sexpr_str(expected, self.repository.converter, _debug=True),
+                mapping_to_sexpr_str(actual, self.repository.converter, _debug=True),
+            )
+
         parameters: dict[str, Any] = {
             "exclude_none": True,
             "exclude_unset": True,
@@ -131,11 +136,9 @@ def assert_model_equal(
         self.assertEqual(
             expected.model_dump(**parameters), actual.model_dump(**parameters), msg=msg
         )
-        if not skip_name_check:
-            # FIXME this shouldn't be optional
-            self.assertEqual(expected.subject_name, actual.subject_name)
-            self.assertEqual(expected.predicate_name, actual.predicate_name)
-            self.assertEqual(expected.object_name, actual.object_name)
+        self.assertEqual(expected.subject_name, actual.subject_name)
+        self.assertEqual(expected.predicate_name, actual.predicate_name)
+        self.assertEqual(expected.object_name, actual.object_name)
 
     def assert_base_model_equal(self, expected: BaseModel, actual: BaseModel) -> None:
         """Check two models are equal by serializing to dict."""
@@ -611,7 +614,7 @@ def test_read(self) -> None:
                     written_path,
                     metadata=TEST_METADATA,
                     exclude_columns=["record_id"],
-                    exclude_prefixes=[MAPPING_HASH_V1_PREFIX],
+                    exclude_prefixes=[MAPPING_HASH_CURIE_PREFIX],
                 )
                 # clean up before actual test
                 db.delete_mapping(example.semantic_mapping)

tests/test_constants.py

@@ -22,7 +22,7 @@
     FORWARDS_MAPS,
     MappingSetRecord,
     SemanticMapping,
-    mapping_hash_v1,
+    hash_mapping_to_reference,
 )
 from sssom_pydantic.constants import (
     MAPPING_SET_SLOTS,
@@ -316,7 +316,7 @@ def test_uniqueness(self) -> None:
         """Test that examples have unique hashes."""
         dd = defaultdict(list)
         for example in EXAMPLES:
-            dd[mapping_hash_v1(example.semantic_mapping, TEST_CONVERTER)].append(example)
+            dd[hash_mapping_to_reference(example.semantic_mapping, TEST_CONVERTER)].append(example)
 
         duplicates = {k: v for k, v in dd.items() if len(v) > 1}
         if duplicates: