DAGE-116: Rename doc_number to number_of_docs (#256)

Author: nseidan

rre-tools/README.md

@@ -45,6 +45,10 @@ uv sync
 uv sync --group dev
 ```
 
+# remove all cached packages
+uv cache clean
+```
+
 ## Running Dataset Generator
 
 Before running the command below, you need to have running search engine instance 

rre-tools/configs/dataset_generator/dataset_generator_config.yaml

@@ -27,7 +27,7 @@ search_engine_url: "http://localhost:8983/solr/"
 #      - "book"
 
 # Maximum number of documents to retrieve from the search engine to generate queries
-doc_number: 2
+number_of_docs: 2
 
 # List of fields from documents used to generate queries and relevance scoring
 # These should match fields available in the search engine schema

rre-tools/docs/dataset_generator/README.md

@@ -27,7 +27,7 @@ one fields, the documents are filtered for both fields (AND-like)
 >       - "fantasy"
 >     - type:
 >       - "book"
-> - **doc_number**: Maximum number of documents to retrieve from the search engine to generate queries (e.g. 100)
+> - **number_of_docs**: Maximum number of documents to retrieve from the search engine to generate queries (e.g. 100)
 > - **doc_fields**: List of fields from documents used to generate queries and relevance scoring. These should match 
 > fields available in the search engine schema and must be at least one field
 >   - an example could be:

rre-tools/src/rre_tools/dataset_generator/config.py

@@ -23,7 +23,7 @@ class Config(BaseModel):
         None,
         description="Optional list of filter conditions for documents"
     )
-    doc_number: int = Field(..., gt=0, description="Number of documents to retrieve from the search engine.")
+    number_of_docs: int = Field(..., gt=0, description="Number of documents to retrieve from the search engine.")
     doc_fields: List[str] = Field(..., min_length=1, description="Fields used for context and scoring.")
     queries: Optional[FilePath] = Field(None, description="Optional file containing predefined queries.")
     generate_queries_from_documents: Optional[bool] = True

rre-tools/src/rre_tools/dataset_generator/main.py

@@ -56,7 +56,7 @@ def generate_and_add_queries(config: Config, data_store: DataStore, llm_service:
     """Retrieve docs and generate queries with LLM Service. Adds docs, queries and ratings to the datastore."""
     docs_to_generate_queries: List[Document] = search_engine.fetch_for_query_generation(
         documents_filter=config.documents_filter,
-        doc_number=config.doc_number,
+        number_of_docs=config.number_of_docs,
         doc_fields=config.doc_fields
     )
 
@@ -68,7 +68,7 @@ def generate_and_add_queries(config: Config, data_store: DataStore, llm_service:
     if remaining == 0:
         return
 
-    num_queries_per_doc: int = int( (remaining // max(1, config.doc_number)) + 1)  # always greater or equal to 1
+    num_queries_per_doc: int = int((remaining // max(1, config.number_of_docs)) + 1)  # always greater or equal to 1
     log.debug(f"Number of documents retrieved for generation: {len(docs_to_generate_queries)}")
     log.debug(f"Pending queries to generate: {remaining}")
     log.debug(f"Number of queries per document: {num_queries_per_doc}")

rre-tools/src/rre_tools/shared/search_engines/elasticsearch_search_engine.py

@@ -46,7 +46,7 @@ def _fetch_all_payload(self) -> Dict[str, Any]:
 
     def fetch_for_query_generation(self,
                                    documents_filter: Union[None, List[Dict[str, List[str]]]],
-                                   doc_number: int,
+                                   number_of_docs: int,
                                    doc_fields: List[str],
                                    start: int = 0) -> List[Document]:
         """
@@ -55,7 +55,7 @@ def fetch_for_query_generation(self,
         Args:
             documents_filter (Union[None, List[Dict[str, List[str]]]]): Optional list of field filters to apply.
                 Each filter is a dictionary mapping field names to allowed values.
-            doc_number (int): Number of documents to retrieve.
+            number_of_docs (int): Number of documents to retrieve.
             doc_fields (List[str]): List of field names to include in the output.
             start (int, optional): Starting index. Defaults to 0.
 
@@ -85,7 +85,7 @@ def fetch_for_query_generation(self,
 
         # Construct the payload (Elasticsearch query body)
         payload = {
-            "size": doc_number,
+            "size": number_of_docs,
             "query": query,
             "from": start,
             "_source": doc_fields

rre-tools/src/rre_tools/shared/search_engines/opensearch_engine.py

@@ -44,7 +44,7 @@ def _fetch_all_payload(self) -> Dict[str, Any]:
 
     def fetch_for_query_generation(self,
                                    documents_filter: Union[None, List[Dict[str, List[str]]]],
-                                   doc_number: int,
+                                   number_of_docs: int,
                                    doc_fields: List[str],
                                    start: int = 0) -> List[Document]:
         """Fetches a list of documents for query generation based on optional filters."""
@@ -75,7 +75,7 @@ def fetch_for_query_generation(self,
             "query": query,
             "_source": fields,
             "from": start,
-            "size": doc_number
+            "size": number_of_docs
         }
 
         return self._search(payload)

rre-tools/src/rre_tools/shared/search_engines/search_engine_base.py

@@ -6,7 +6,7 @@
 from pydantic import HttpUrl
 from rre_tools.shared.models.document import Document
 
-DOC_NUMBER_EACH_FETCH = 100
+NUMBER_OF_DOCS_EACH_FETCH = 100
 
 class BaseSearchEngine(ABC):
     def __init__(self, endpoint: HttpUrl):
@@ -31,18 +31,18 @@ def fetch_all(self, doc_fields: List[str]) -> Iterator[Document]:
         while start < total_hits:
             batch = self.fetch_for_query_generation(
                 documents_filter=None,
-                doc_number=DOC_NUMBER_EACH_FETCH,
+                number_of_docs=NUMBER_OF_DOCS_EACH_FETCH,
                 doc_fields=doc_fields,
                 start=start
                 )
             if not batch:
                 break
             for doc in batch:
                 yield doc
-            # if we didn't reach the end of the docs, then len(batch) == DOC_NUMBER_EACH_FETCH
-            # if we reached the end of the docs. then len(batch) <= DOC_NUMBER_EACH_FETCH -> next iteration we exit the
-            # loop since we are adding DOC_NUMBER_EACH_FETCH (not len(batch)) and start becomes greater than total_hits
-            start += DOC_NUMBER_EACH_FETCH
+            # if we didn't reach the end of the docs, then len(batch) == NUMBER_OF_DOCS_EACH_FETCH if we reached the
+            # end of the docs. then len(batch) <= NUMBER_OF_DOCS_EACH_FETCH -> next iteration we exit the loop since
+            # we are adding NUMBER_OF_DOCS_EACH_FETCH (not len(batch)) and start becomes greater than total_hits
+            start += NUMBER_OF_DOCS_EACH_FETCH
 
 
     def _parse_query_template(self, path: Path | str) -> Dict[str, Any]:
@@ -72,7 +72,7 @@ def _replace_placeholder(self, obj: Any, placeholder: str, keyword: str | None)
     @abstractmethod
     def fetch_for_query_generation(self,
                                    documents_filter: Union[None, List[Dict[str, List[str]]]],
-                                   doc_number: int,
+                                   number_of_docs: int,
                                    doc_fields: List[str],
                                    start: int = 0) \
             -> List[Document]:

rre-tools/src/rre_tools/shared/search_engines/solr_search_engine.py

@@ -57,22 +57,22 @@ def _get_total_hits(self, payload: Dict[str, Any]) -> int:
 
     def fetch_for_query_generation(self,
                                    documents_filter: Union[None, List[Dict[str, List[str]]]],
-                                   doc_number: int, doc_fields: List[str], start: int = 0) \
+                                   number_of_docs: int, doc_fields: List[str], start: int = 0) \
             -> List[Document]:
         """
         Fetches a set of documents from Solr for the purpose of query generation.
 
         Args:
             documents_filter (Union[None, List[Dict[str, List[str]]]]): Optional filter constraints for fields and their allowed values.
-            doc_number (int): Number of documents to retrieve.
+            number_of_docs (int): Number of documents to retrieve.
             doc_fields (List[str]): List of field names to include in the output.
             start (int, optional): Starting index of the query. Defaults to 0.
 
         Returns:
             List[Document]: A list of retrieved documents as `Document` objects.
         """
         payload: Dict[str, Any] = self._fetch_all_payload
-        payload['rows'] = doc_number
+        payload['rows'] = number_of_docs
         payload['start'] = start
         payload['fl'] = self._unify_fields(doc_fields)
 

rre-tools/src/rre_tools/shared/search_engines/vespa_search_engine.py

@@ -131,7 +131,7 @@ def _normalize_field_value(v: Any) -> List[str]:
     def fetch_for_query_generation(
         self,
         documents_filter: Union[None, List[Dict[str, List[str]]]],
-        doc_number: int,
+        number_of_docs: int,
         doc_fields: Optional[List[str]],
         start: int  = 0,
     ) -> List[Document]:
@@ -140,7 +140,7 @@ def fetch_for_query_generation(
 
         Args:
             documents_filter: Optional list of filter dictionaries for query restriction.
-            doc_number: Number of documents to retrieve.
+            number_of_docs: Number of documents to retrieve.
             doc_fields: Optional list of fields to include in the response.
             start: Optional start index to retrieve documents from.
 
@@ -155,7 +155,7 @@ def fetch_for_query_generation(
         yql = self._build_yql(doc_fields or [], where)
 
         payload["yql"] = yql
-        payload["hits"] = int(doc_number)
+        payload["hits"] = int(number_of_docs)
         payload["presentation.format"] =  "json"
         payload['offset'] = start