feat/knowledge-source

.github/CONTRIBUTING.md

@@ -34,7 +34,7 @@ This project and everyone participating in it is governed by our Code of Conduct
    ```bash
    # Create a virtual environment
    python -m venv venv
-   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   knowledge venv/bin/activate  # On Windows: venv\Scripts\activate
 
    # Install dependencies
    pip install poetry

pilottai_tools/source/knowledge.py → pilottai_tools/knowledge/knowledge.py

@@ -8,7 +8,7 @@
 from pydantic import BaseModel, Field
 
 
-from pilottai.config.model import KnowledgeSource, CacheEntry
+from pilottai_tools.config.model import KnowledgeSource, CacheEntry
 
 class DataManager:
     def __init__(self, cache_size: int = 1000, cache_ttl: int = 3600):
@@ -58,7 +58,7 @@ async def add_source(self, source: KnowledgeSource) -> bool:
             return connected
 
         except Exception as e:
-            self.logger.error(f"Error adding source {source.name}: {str(e)}")
+            self.logger.error(f"Error adding knowledge {source.name}: {str(e)}")
             return False
 
     async def query_knowledge(
@@ -89,7 +89,7 @@ async def query_knowledge(
                                 if result is not None:
                                     results.append(result)
                         except Exception as e:
-                            self.logger.error(f"Error querying source {source_type}: {str(e)}")
+                            self.logger.error(f"Error querying knowledge {source_type}: {str(e)}")
                             source.error_count += 1
                             continue
                 if results:
@@ -119,12 +119,12 @@ async def _query_source_with_retry(
                     return result
             except asyncio.TimeoutError:
                 self.logger.warning(
-                    f"Query timeout for source {source.name}, attempt {attempt + 1}"
+                    f"Query timeout for knowledge {source.name}, attempt {attempt + 1}"
                 )
                 source.error_count += 1
             except Exception as e:
                 self.logger.error(
-                    f"Query failed for source {source.name}, attempt {attempt + 1}: {str(e)}"
+                    f"Query failed for knowledge {source.name}, attempt {attempt + 1}: {str(e)}"
                 )
                 source.error_count += 1
                 if attempt < source.max_retries - 1:
@@ -223,7 +223,7 @@ async def cleanup(self):
                 ]
                 for k in expired_keys:
                     del self.cache[k]
-                # Check source health
+                # Check knowledge health
                 for source_name, source in self.sources.items():
                     if source.error_count > source.max_retries:
                         source.is_connected = False

pilottai_tools/source/source/audio_input.py → pilottai_tools/knowledge/source/audio_input.py

@@ -5,12 +5,12 @@
 from datetime import datetime, timedelta
 
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class AudioInput(BaseInputSource):
     """
-    Input source for processing audio files.
+    Input knowledge for processing audio files.
     Extracts and processes speech from audio files using speech-to-text technology.
     """
 

pilottai_tools/source/source/base_input.py → pilottai_tools/knowledge/source/base_input.py

@@ -8,7 +8,7 @@
 
 
 class SourceMetadata(BaseModel):
-    """Metadata for an input source"""
+    """Metadata for an input knowledge"""
     source_type: str
     created_at: datetime = Field(default_factory=datetime.now)
     updated_at: datetime = Field(default_factory=datetime.now)
@@ -19,7 +19,7 @@ class SourceMetadata(BaseModel):
 
 class BaseInputSource(ABC):
     """
-    Abstract base class for all source input sources.
+    Abstract base class for all knowledge input sources.
     Provides common functionality for processing and storing content.
     """
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -62,7 +62,7 @@ def __init__(
         self.logger = self._setup_logger()
 
     def _setup_logger(self) -> logging.Logger:
-        """Setup a logger for this input source"""
+        """Setup a logger for this input knowledge"""
         logger = logging.getLogger(f"InputSource_{self.name}")
         if not logger.handlers:
             handler = logging.StreamHandler()
@@ -77,36 +77,36 @@ def _setup_logger(self) -> logging.Logger:
     @abstractmethod
     async def connect(self) -> bool:
         """
-        Establish a connection to the source.
+        Establish a connection to the knowledge.
         Returns True if successful, False otherwise.
         """
         pass
 
     @abstractmethod
     async def query(self, query: str) -> Any:
         """
-        Query the source with the given query.
+        Query the knowledge with the given query.
         This method should be implemented by subclasses.
         """
         pass
 
     @abstractmethod
     async def validate_content(self) -> bool:
         """
-        Validate that the content from the source is accessible and processable.
+        Validate that the content from the knowledge is accessible and processable.
         Returns True if valid, False otherwise.
         """
         pass
 
     async def add(self) -> bool:
         """
-        Process content from the source, chunk it, and save it to storage.
+        Process content from the knowledge, chunk it, and save it to storage.
         Returns True if successful, False otherwise.
         """
         try:
             # Validate content
             if not await self.validate_content():
-                self.logger.error(f"Content validation failed for source {self.name}")
+                self.logger.error(f"Content validation failed for knowledge {self.name}")
                 return False
 
             # Process and chunk content
@@ -119,14 +119,14 @@ async def add(self) -> bool:
             return len(self.chunks) > 0
 
         except Exception as e:
-            self.logger.error(f"Error adding content from source {self.name}: {str(e)}")
+            self.logger.error(f"Error adding content from knowledge {self.name}: {str(e)}")
             self.error_count += 1
             return False
 
     @abstractmethod
     async def _process_content(self) -> None:
         """
-        Process the content from the source and populate the chunks.
+        Process the content from the knowledge and populate the chunks.
         This method should be implemented by subclasses.
         """
         pass
@@ -139,7 +139,7 @@ async def _save_to_storage(self) -> bool:
 
             # Create metadata for each chunk
             chunk_metadata = [{
-                "source": self.name,
+                "knowledge": self.name,
                 "collection": self.collection_name,
                 "chunk_index": i,
                 "total_chunks": len(self.chunks),
@@ -169,12 +169,12 @@ def _chunk_text(self, text: str) -> List[str]:
         return chunks
 
     async def refresh(self) -> bool:
-        """Refresh content from the source"""
+        """Refresh content from the knowledge"""
         self.chunks = []
         return await self.add()
 
     def get_info(self) -> Dict[str, Any]:
-        """Get information about this input source"""
+        """Get information about this input knowledge"""
         return {
             "name": self.name,
             "type": self.__class__.__name__,

pilottai_tools/source/source/doc_input.py → pilottai_tools/knowledge/source/doc_input.py

@@ -4,12 +4,12 @@
 import docx
 import io
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class DocInput(BaseInputSource):
     """
-    Input source for processing Microsoft Word documents (.doc, .docx).
+    Input knowledge for processing Microsoft Word documents (.doc, .docx).
     Extracts and processes text content from Word documents.
     """
 

pilottai_tools/source/source/image_input.py → pilottai_tools/knowledge/source/image_input.py

@@ -3,17 +3,16 @@
 from datetime import datetime
 import io
 from PIL import Image
-import pytesseract
 import numpy as np
-import cv2.cv2 as cv2
+import cv2 as cv2
 
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class ImageInput(BaseInputSource):
     """
-    Input source for processing images.
+    Input knowledge for processing images.
     Extracts text content from images using OCR (Optical Character Recognition).
     """
 
@@ -69,7 +68,7 @@ async def connect(self) -> bool:
                 self.is_connected = True
                 return True
 
-            self.logger.error("No image source provided")
+            self.logger.error("No image knowledge provided")
             self.is_connected = False
             return False
 
@@ -138,12 +137,12 @@ async def extract_text(self) -> bool:
             if self.preprocess:
                 image_for_ocr = self._preprocess_image(self.pil_image)
 
-            # Run OCR
-            self.text_content = pytesseract.image_to_string(
-                image_for_ocr,
-                lang=self.lang,
-                config=self.ocr_config
-            )
+            #TODO
+            # self.text_content = pytesseract.image_to_string(
+            #     image_for_ocr,
+            #     lang=self.lang,
+            #     config=self.ocr_config
+            # )
 
             return bool(self.text_content.strip())
 

pilottai_tools/source/source/json_input.py → pilottai_tools/knowledge/source/json_input.py

@@ -2,13 +2,13 @@
 from typing import Any, Dict, List, Optional, Union
 from datetime import datetime
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class JSONInput(BaseInputSource):
     """
-    Input source for processing JSON data.
-    Handles structured JSON content for source extraction.
+    Input knowledge for processing JSON data.
+    Handles structured JSON content for knowledge extraction.
     """
 
     def __init__(
@@ -148,7 +148,7 @@ async def _process_content(self) -> None:
             text_content = json.dumps(self.json_data, indent=2)
 
         self.chunks = self._chunk_text(text_content)
-        self.logger.info(f"Created {len(self.chunks)} chunks from JSON source {self.name}")
+        self.logger.info(f"Created {len(self.chunks)} chunks from JSON knowledge {self.name}")
 
     def _flatten_json(self, data, parent_key='', sep='.') -> str:
         """Flatten nested JSON into a string representation"""

pilottai_tools/source/source/markup_input.py → pilottai_tools/knowledge/source/markup_input.py

@@ -8,12 +8,12 @@
 import json
 from xml.etree import ElementTree as ET
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class MarkupInput(BaseInputSource):
     """
-    Input source for processing markup documents (HTML, XML, Markdown, YAML).
+    Input knowledge for processing markup documents (HTML, XML, Markdown, YAML).
     Extracts and processes content from various markup formats.
     """
 
@@ -80,7 +80,7 @@ async def connect(self) -> bool:
                 self.is_connected = bool(self.raw_content)
                 return self.is_connected
 
-            self.logger.error("No content source provided")
+            self.logger.error("No content knowledge provided")
             self.is_connected = False
             return False
 

pilottai_tools/source/source/stream_input.py → pilottai_tools/knowledge/source/stream_input.py

@@ -6,12 +6,12 @@
 import threading
 from collections import deque
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class StreamInput(BaseInputSource):
     """
-    Input source for processing streaming data.
+    Input knowledge for processing streaming data.
     Handles continuous data streams and real-time processing.
     """
 
@@ -62,7 +62,7 @@ def __init__(
             self.start()
 
     async def connect(self) -> bool:
-        """Check if the streaming source is accessible"""
+        """Check if the streaming knowledge is accessible"""
         try:
             # For streaming sources, connection is established by starting the worker
             if self.running:

pilottai_tools/source/source/string_input.py → pilottai_tools/knowledge/source/string_input.py

@@ -1,12 +1,12 @@
 from typing import Any, Optional
 from datetime import datetime
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class StringInput(BaseInputSource):
     """
-    Input source for processing plain text strings.
+    Input knowledge for processing plain text strings.
     Implements base functionality for text content handling.
     """
 
@@ -74,7 +74,7 @@ async def query(self, query: str) -> Any:
     async def validate_content(self) -> bool:
         """Validate that the string content is not empty"""
         if not self.text:
-            self.logger.warning(f"No text content for source {self.name}")
+            self.logger.warning(f"No text content for knowledge {self.name}")
             return False
         return True
 
@@ -84,7 +84,7 @@ async def _process_content(self) -> None:
             return
 
         self.chunks = self._chunk_text(self.text)
-        self.logger.info(f"Created {len(self.chunks)} chunks from string source {self.name}")
+        self.logger.info(f"Created {len(self.chunks)} chunks from string knowledge {self.name}")
 
     def set_text(self, text: str) -> None:
         """Update the text content"""

pilottai_tools/source/source/structured_input.py → pilottai_tools/knowledge/source/structured_input.py

@@ -4,13 +4,13 @@
 from datetime import datetime
 import io
 
-from pilottai.knowledge.source.base_input import BaseInputSource
+from pilottai_tools.knowledge.source.base_input import BaseInputSource
 
 
 class StructuredInput(BaseInputSource):
     """
-    Input source for processing structured data like CSV, Excel, or memory tables.
-    Converts structured data into a text representation for source extraction.
+    Input knowledge for processing structured data like CSV, Excel, or memory tables.
+    Converts structured data into a text representation for knowledge extraction.
     """
 
     def __init__(
@@ -85,7 +85,7 @@ async def connect(self) -> bool:
                 self.is_connected = True
                 return True
 
-            self.logger.error("No data source provided")
+            self.logger.error("No data knowledge provided")
             self.is_connected = False
             return False
 
@@ -112,7 +112,7 @@ async def query(self, query: str) -> Any:
         """Query the structured data"""
         if not self.is_connected or self.dataframe is None:
             if not await self.connect():
-                raise ValueError("Could not connect to data source")
+                raise ValueError("Could not connect to data knowledge")
 
         self.access_count += 1
         self.last_access = datetime.now()