fix(file_processors): expose Docling structure as attributes

Serialize headings and page numbers as scalar chunk attributes so vector store search can return them without a separate metadata field. Preserve the existing legacy fallback shape for backward compatibility.

Refs #6396

Signed-off-by: Peter Gustafsson <peter.gustafsson6@gmail.com>

Author: Peter Gustafsson

src/ogx/providers/inline/file_processor/docling/_metadata.py

@@ -6,24 +6,26 @@
 
 from typing import Any
 
+from ogx.providers.utils.files.structural_metadata import structural_metadata_as_attributes
+
 
 def extract_structural_metadata(doc_chunk: Any) -> dict[str, Any]:
     chunk_meta = getattr(doc_chunk, "meta", None)
     if chunk_meta is None:
         return {}
 
-    metadata: dict[str, Any] = {}
     headings = getattr(chunk_meta, "headings", None)
-    if headings:
-        metadata["headings"] = headings
-
+    legacy_headings = getattr(doc_chunk, "headings", None)
     page_numbers = {
         page_number
         for doc_item in getattr(chunk_meta, "doc_items", [])
         for provenance in (getattr(doc_item, "prov", None) or [])
         if (page_number := getattr(provenance, "page_no", None)) is not None
     }
-    if page_numbers:
-        metadata["page_numbers"] = sorted(page_numbers)
-
+    metadata: dict[str, Any] = structural_metadata_as_attributes(
+        headings=headings,
+        page_numbers=sorted(page_numbers),
+    )
+    if not headings and legacy_headings:
+        metadata["headings"] = legacy_headings
     return metadata

src/ogx/providers/remote/file_processor/docling_serve/docling_serve.py

@@ -19,6 +19,7 @@
 
 from ogx.log import get_logger
 from ogx.providers.utils.files.response import response_body_bytes
+from ogx.providers.utils.files.structural_metadata import structural_metadata_as_attributes
 from ogx.providers.utils.vector_io.vector_utils import generate_chunk_id
 from ogx_api.common.errors import InvalidParameterError
 from ogx_api.file_processors import ProcessFileRequest, ProcessFileResponse
@@ -344,13 +345,17 @@ async def _convert_and_chunk(
             }
 
             legacy_meta = raw_chunk.get("meta") or {}
-            headings = raw_chunk.get("headings") or legacy_meta.get("headings")
-            if headings:
-                meta["headings"] = headings
-
+            headings = raw_chunk.get("headings")
+            legacy_headings = legacy_meta.get("headings")
             page_numbers = raw_chunk.get("page_numbers") or legacy_meta.get("page_numbers")
-            if page_numbers:
-                meta["page_numbers"] = page_numbers
+            meta.update(
+                structural_metadata_as_attributes(
+                    headings=headings,
+                    page_numbers=page_numbers,
+                )
+            )
+            if not headings and legacy_headings:
+                meta["headings"] = legacy_headings
 
             chunks.append(
                 Chunk(
@@ -432,13 +437,17 @@ async def _convert_and_chunk_async(
             }
 
             legacy_meta = getattr(raw_chunk, "meta", None)
-            headings = getattr(raw_chunk, "headings", None) or getattr(legacy_meta, "headings", None)
-            if headings:
-                meta["headings"] = headings
-
+            headings = getattr(raw_chunk, "headings", None)
+            legacy_headings = getattr(legacy_meta, "headings", None)
             page_numbers = getattr(raw_chunk, "page_numbers", None) or getattr(legacy_meta, "page_numbers", None)
-            if page_numbers:
-                meta["page_numbers"] = page_numbers
+            meta.update(
+                structural_metadata_as_attributes(
+                    headings=headings,
+                    page_numbers=page_numbers,
+                )
+            )
+            if not headings and legacy_headings:
+                meta["headings"] = legacy_headings
 
             chunks.append(
                 Chunk(

src/ogx/providers/utils/files/structural_metadata.py

@@ -0,0 +1,28 @@
+# Copyright (c) The OGX Contributors.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from typing import Any
+
+
+def _attribute_value(value: Any, separator: str) -> str:
+    if isinstance(value, list | tuple):
+        return separator.join(str(item) for item in value if str(item))
+    return str(value) if value is not None else ""
+
+
+def structural_metadata_as_attributes(*, headings: Any = None, page_numbers: Any = None) -> dict[str, str]:
+    """Convert structural chunk metadata to scalar vector-store attributes."""
+    metadata: dict[str, str] = {}
+
+    headings_value = _attribute_value(headings, " > ")
+    if headings_value:
+        metadata["headings"] = headings_value
+
+    page_numbers_value = _attribute_value(page_numbers, ", ")
+    if page_numbers_value:
+        metadata["page_numbers"] = page_numbers_value
+
+    return metadata

tests/unit/providers/file_processor/test_docling_metadata.py

@@ -7,6 +7,8 @@
 from types import SimpleNamespace
 
 from ogx.providers.inline.file_processor.docling._metadata import extract_structural_metadata
+from ogx.providers.utils.files.structural_metadata import structural_metadata_as_attributes
+from ogx_api.vector_io import VectorStoreContent, VectorStoreSearchResponse
 
 
 def test_extract_structural_metadata_from_native_docling_chunk():
@@ -23,12 +25,63 @@ def test_extract_structural_metadata_from_native_docling_chunk():
     )
 
     assert extract_structural_metadata(doc_chunk) == {
-        "headings": ["Introduction", "Architecture"],
-        "page_numbers": [1, 2],
+        "headings": "Introduction > Architecture",
+        "page_numbers": "1, 2",
     }
 
 
 def test_extract_structural_metadata_omits_empty_values():
     doc_chunk = SimpleNamespace(meta=SimpleNamespace(headings=None, doc_items=[]))
 
     assert extract_structural_metadata(doc_chunk) == {}
+
+
+def test_extract_structural_metadata_preserves_legacy_top_level_headings():
+    doc_chunk = SimpleNamespace(
+        headings=["Legacy heading"],
+        meta=SimpleNamespace(headings=None, doc_items=[]),
+    )
+
+    assert extract_structural_metadata(doc_chunk) == {
+        "headings": ["Legacy heading"],
+    }
+
+
+def test_structural_metadata_attributes_keep_commas_inside_headings():
+    assert structural_metadata_as_attributes(
+        headings=["Safety, Security", "Database setup"],
+        page_numbers=[4, 5],
+    ) == {
+        "headings": "Safety, Security > Database setup",
+        "page_numbers": "4, 5",
+    }
+
+
+def test_structural_metadata_attributes_preserve_existing_strings():
+    assert structural_metadata_as_attributes(
+        headings="Database setup",
+        page_numbers="4, 5",
+    ) == {
+        "headings": "Database setup",
+        "page_numbers": "4, 5",
+    }
+
+
+def test_structural_metadata_is_valid_in_vector_store_search_attributes():
+    metadata = structural_metadata_as_attributes(
+        headings=["Installation", "Database setup"],
+        page_numbers=[4, 5],
+    )
+
+    result = VectorStoreSearchResponse(
+        file_id="file-123",
+        filename="manual.pdf",
+        score=1.0,
+        attributes=metadata,
+        content=[VectorStoreContent(type="text", text="Database setup instructions")],
+    )
+
+    assert result.attributes == {
+        "headings": "Installation > Database setup",
+        "page_numbers": "4, 5",
+    }

tests/unit/providers/file_processor/test_docling_serve.py

@@ -61,7 +61,7 @@ def _make_httpx_response(json_body: dict, status_code: int = 200) -> httpx.Respo
             "filename": "test.pdf",
             "chunk_index": 2,
             "text": "Third chunk of text.",
-            "headings": ["Conclusion"],
+            "headings": ["Safety, Security", "Conclusion"],
             "doc_items": ["#/texts/2"],
             "page_numbers": [2, 3],
         },
@@ -232,12 +232,30 @@ async def test_structural_metadata_propagated(self, processor: DoclingServeFileP
         with patch("httpx.AsyncClient.post", return_value=_make_httpx_response(CHUNK_RESPONSE)):
             response = await processor.process_file(request, file=upload_file)
 
-        assert response.chunks[0].metadata["headings"] == ["Introduction"]
-        assert response.chunks[0].metadata["page_numbers"] == [1]
+        assert response.chunks[0].metadata["headings"] == "Introduction"
+        assert response.chunks[0].metadata["page_numbers"] == "1"
         assert "headings" not in response.chunks[1].metadata
         assert "page_numbers" not in response.chunks[1].metadata
-        assert response.chunks[2].metadata["headings"] == ["Conclusion"]
-        assert response.chunks[2].metadata["page_numbers"] == [2, 3]
+        assert response.chunks[2].metadata["headings"] == "Safety, Security > Conclusion"
+        assert response.chunks[2].metadata["page_numbers"] == "2, 3"
+
+    async def test_legacy_nested_headings_keep_existing_list_type(
+        self, processor: DoclingServeFileProcessor, upload_file: UploadFile
+    ):
+        request = ProcessFileRequest(chunking_strategy=VectorStoreChunkingStrategyAuto())
+        legacy_response = {
+            "chunks": [
+                {
+                    "text": "Legacy chunk shape.",
+                    "meta": {"headings": ["Legacy heading"]},
+                }
+            ]
+        }
+
+        with patch("httpx.AsyncClient.post", return_value=_make_httpx_response(legacy_response)):
+            response = await processor.process_file(request, file=upload_file)
+
+        assert response.chunks[0].metadata["headings"] == ["Legacy heading"]
 
     async def test_chunk_window_set(self, processor: DoclingServeFileProcessor, upload_file: UploadFile):
         request = ProcessFileRequest(chunking_strategy=VectorStoreChunkingStrategyAuto())
@@ -514,6 +532,6 @@ async def test_local_docker_allows_chunking(self, upload_file: UploadFile):
 
         assert result.chunks is not None
         assert len(result.chunks) > 0
-        assert result.chunks[0].metadata["headings"] == ["Introduction"]
-        assert result.chunks[0].metadata["page_numbers"] == [1, 2]
+        assert result.chunks[0].metadata["headings"] == "Introduction"
+        assert result.chunks[0].metadata["page_numbers"] == "1, 2"
         assert result.metadata["conversion_method"] == "async"