Respond to Pablo's feedback

Author: nseidan

rre-dataset-generator/src/search_engine/opensearch_engine.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from typing import List, Dict, Any, Union
 
@@ -99,15 +100,27 @@ def _search(self, payload: Dict[str, Any]) -> List[Document]:
         return result
 
     @staticmethod
-    def _normalize(value: Any) -> Any:
-        """
-        Normalize a field value:
-        - string → return [cleaned string]
-        - list of strings → return [cleaned strings]
-        - else → return as it is
-        """
-        if isinstance(value, str):
-            return [clean_text(value)]
-        if isinstance(value, list) and all(isinstance(i, str) for i in value):
-            return [clean_text(i) for i in value]
-        return value
+    def _normalize(value: Any) -> List[str]:
+        """Normalize a field value into a list of cleaned strings or throws an exception."""
+        try:
+            if value is None:
+                return []
+
+            if isinstance(value, str):
+                return [clean_text(value)]
+
+            if isinstance(value, list):
+                return [clean_text(v) if isinstance(v, str) else str(v) for v in value]
+
+            if isinstance(value, dict):
+                cleaned_dict = {
+                    k: clean_text(v) if isinstance(v, str) else v
+                    for k, v in value.items()
+                }
+                return [json.dumps(cleaned_dict)]
+
+            return [str(value)]
+
+        except Exception as e:
+            raise ValueError(f"Failed to normalize value: {value}") from e
+

rre-dataset-generator/tests/mocks/opensearch.py

@@ -1,16 +1,27 @@
+from typing import List, Dict, Union, Any
+
 import requests
 
 
 class MockResponseOpenSearchEngine:
-    def __init__(self, hits_data, status_code=200):
-        self._hits_data = hits_data if isinstance(hits_data, list) else [hits_data]
+    def __init__(self, hits_data: Union[Dict[str, Any], List[Dict[str, Any]]], status_code: int = 200):
+        if isinstance(hits_data, dict):
+            self._hits_data = [hits_data]
+        elif isinstance(hits_data, list):
+            self._hits_data = hits_data
+        else:
+            raise ValueError("hits_data must be a dict or a list of dicts")
+
+        if not isinstance(status_code, int):
+            raise TypeError("status_code must be an int")
+
         self.status_code = status_code
 
-    def raise_for_status(self):
+    def raise_for_status(self) -> None:
         if self.status_code != 200:
             raise requests.exceptions.HTTPError(f"Status code: {self.status_code}")
 
-    def json(self):
+    def json(self) -> Dict[str, Any]:
         return {
             "hits": {
                 "total": {"value": len(self._hits_data), "relation": "eq"},

rre-dataset-generator/tests/unit/test_opensearch_engine.py

@@ -1,5 +1,7 @@
+import json
 import logging
 
+import pytest
 import requests
 
 from src.config import Config
@@ -11,47 +13,84 @@
 configure_logging(level=logging.DEBUG)
 
 
-def test_opensearch_search_engine(monkeypatch):
-    config = Config.load("tests/unit/resources/good_config_opensearch.yaml")
-    search_engine = OpenSearchEngine("http://testurl/testcore")
+@pytest.fixture
+def opensearch_config():
+    """Fixture that loads a valid OpenSearch config for unit tests."""
+    return Config.load("tests/unit/resources/good_config_opensearch.yaml")
 
-    doc = {
-        "id": "1",
-        "title": "test title",
-        "description": "test description"
-    }
 
-    hit = {
+@pytest.fixture
+def opensearch_hit():
+    return {
         "_index": "testcore",
         "_id": "1",
         "_score": 1.0,
-        "_source": doc
+        "_source": {
+            "id": "1",
+            "title": "test title",
+            "description": "test description"
+        }
     }
 
+
+def test_fetch_for_query_generation(monkeypatch, opensearch_config, opensearch_hit):
+    opensearch = OpenSearchEngine("http://testurl/testcore")
+
     expected_doc = Document(
-        id=doc["id"],
-        fields={k: search_engine._normalize(v) for k, v in doc.items() if k != "id"}
+        id="1",
+        fields={
+            "title": opensearch._normalize("test title"),
+            "description": opensearch._normalize("test description")
+        }
+    )
+
+    def mock_post(*args, **kwargs):
+        return MockResponseOpenSearchEngine([opensearch_hit], status_code=200)
+
+    monkeypatch.setattr(requests, "post", mock_post)
+
+    result = opensearch.fetch_for_query_generation(
+        documents_filter=opensearch_config.documents_filter,
+        doc_number=opensearch_config.doc_number,
+        doc_fields=opensearch_config.doc_fields
     )
 
-    def post(*args, **kwargs):
-        return MockResponseOpenSearchEngine([hit], status_code=200)
+    assert len(result) == 1, "Expected one document"
+    assert result[0] == expected_doc, "Mismatch in query generated doc"
 
-    monkeypatch.setattr(requests, "post", post)
 
-    result = search_engine.fetch_for_query_generation(
-        documents_filter=config.documents_filter,
-        doc_number=config.doc_number,
-        doc_fields=config.doc_fields
+def test_fetch_for_evaluation(monkeypatch, opensearch_config, opensearch_hit):
+    opensearch = OpenSearchEngine("http://testurl/testcore")
+
+    expected_doc = Document(
+        id="1",
+        fields={
+            "title": opensearch._normalize("test title"),
+            "description": opensearch._normalize("test description")
+        }
     )
 
-    assert len(result) == 1
-    assert result[0] == expected_doc
+    def mock_post(*args, **kwargs):
+        return MockResponseOpenSearchEngine([opensearch_hit], status_code=200)
 
-    result = search_engine.fetch_for_evaluation(
+    monkeypatch.setattr(requests, "post", mock_post)
+
+    result = opensearch.fetch_for_evaluation(
+        query_template=opensearch_config.query_template,
         keyword="car",
-        query_template=config.query_template,
-        doc_fields=config.doc_fields
+        doc_fields=opensearch_config.doc_fields
     )
 
-    assert len(result) == 1
-    assert result[0] == expected_doc
+    assert len(result) == 1, "Expected one document"
+    assert result[0] == expected_doc, "Mismatch in doc evaluation"
+
+
+def test_normalize():
+    engine = OpenSearchEngine("http://dummy")
+
+    assert engine._normalize("  hello  ") == ["hello"], "Failed to normalize string"
+    assert engine._normalize(["a", " b "]) == ["a", "b"], "Failed to normalize list of strings"
+    assert engine._normalize(123) == ["123"], "Failed to normalize integer"
+    assert engine._normalize(None) == [], "Failed to normalize None"
+    assert engine._normalize({"key": "   value  "}) == [json.dumps({"key": "value"})], "Failed to normalize dict"
+