chg: fts text

Author: nicokant

pyproject.toml

@@ -18,7 +18,8 @@ dependencies = [
   "typer>=0.20.0",
   "pygeometa>=0.19.0",
   "orjson>=3.11.4",
-  "duckdb>=1.4.2"
+  "duckdb>=1.4.2",
+  "lxml>=6.0.2"
 ]
 description = ""
 license = "GPL-3.0+"

src/datasync/dms.py

@@ -3,6 +3,7 @@
 import typer
 from pygeometa.schemas.iso19139 import ISO19139OutputSchema
 
+from .libs.helpers import get_anytext
 from .settings import log
 
 app = typer.Typer()
@@ -30,6 +31,9 @@ def generate_csw_metadata(
     conn = duckdb.connect()
     conn.sql("INSTALL yaml FROM community; LOAD yaml; Install spatial; load spatial")
 
+    conn.create_function("to_iso19139", to_iso19139)
+    conn.create_function("xml_bag", get_anytext)
+
     datasets = conn.read_parquet(base_url + "datasets_dataset.parquet").filter(
         "json_keys(metadata) <> []"
     )
@@ -219,9 +223,11 @@ def generate_csw_metadata(
     )
     log.debug(res)
 
-    conn.create_function("to_iso19139", to_iso19139)
+    iso_xml = res.select("id, to_iso19139(metadata) as xml")
+
+    log.debug(iso_xml)
 
-    log.debug(res.select("id, to_iso19139(metadata) as xml"))
+    log.debug(iso_xml.select("*, xml_bag(xml) as fts_text"))
 
 
 if __name__ == "__main__":

src/datasync/libs/helpers.py

@@ -0,0 +1,25 @@
+import re
+
+from lxml import etree
+
+PARSER = etree.XMLParser(resolve_entities=False)
+
+
+def get_anytext(bag: str) -> str:
+    """
+    generate bag of text for free text searches
+    accepts list of words, string of XML, or etree.Element
+    """
+
+    if isinstance(bag, list):  # list of words
+        return " ".join([_f for _f in bag if _f]).strip()
+    else:  # xml
+        if isinstance(bag, bytes) or isinstance(bag, str):
+            # serialize to lxml
+            bag = etree.fromstring(bag, PARSER)  # noqa: S320
+        # get all XML element content
+        return re.sub(
+            r"\s+",
+            " ",
+            " ".join([value.strip() for value in bag.xpath("//text()")]).strip(),
+        )