adding summary_overlap config variable

Author: Lucas-Fernandes-Martins

configs/config.yaml

@@ -52,7 +52,8 @@ generation:
   num_cot_enhance_examples: null  # Maximum number of conversations to enhance (null = enhance all)
   batch_size: 32     # Number of requests to batch together (for create)
   max_context_length: 8000       # Context Length of the MODEL. Useful while Generating Summary
-
+  summary_overlap: 0       # Overlap between chunks to maintain context. Useful while Generating Summary
+  
 # Content curation parameters
 curate:
   threshold: 7.0     # Default quality threshold (1-10)

synthetic_data_kit/generators/qa_generator.py

@@ -39,16 +39,17 @@ def generate_summary(self,
         if verbose:
             print("Generating document summary...")
 
-        # Get summary prompt from config
+        # Get summary prompt and params from config
         prompt = get_prompt(self.config, "summary")
         max_context_length = self.generation_config.get("max_context_length", 8000)
+        summary_overlap = self.generation_config.get("summary_overlap", 0)
 
         if rolling_summary:
             summary_per_chunk = []
             #split text into long chunks for summarization
             chunks = split_into_chunks(document_text,
                                        chunk_size=max_context_length,
-                                       overlap=0)
+                                       overlap=summary_overlap)
 
             for chunk in chunks:
                 messages = [