config.yaml.example

sources:
  - type: "s3"
    name: "account1"
    #enabled: true  # optional; omit or set true to enable, false to skip entirely
    config:
      endpoint: "${S3_ACCOUNT1_ENDPOINT}"
      access_key: "${S3_ACCOUNT1_ACCESS_KEY}"
      secret_key: "${S3_ACCOUNT1_SECRET_KEY}"
      region: "${S3_ACCOUNT1_REGION}"
      use_ssl: "${S3_ACCOUNT1_USE_SSL}"
      buckets: "${S3_ACCOUNT1_BUCKETS}" # comma-separated string or list
      schedules: "${S3_ACCOUNT1_SCHEDULES}"
      #request_delay: 0  # optional, delay in seconds between items (default: 0)

  #- type: "directory"
  #  name: "local_docs"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    path: "${DIRECTORY1_PATH}"
  #    recursive: true
  #    required_exts: "txt,md,pdf"
  #    exclude_hidden: true
  #    exclude_empty: false
  #    num_files_limit: 1000
  #    schedules: "${DIRECTORY1_SCHEDULES}"
  #    #request_delay: 0  # optional, delay in seconds between items (default: 0)

  #- type: "s3"
  #  name: "account2"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    endpoint: "${S3_ACCOUNT2_ENDPOINT}"
  #    access_key: "${S3_ACCOUNT2_ACCESS_KEY}"
  #    secret_key: "${S3_ACCOUNT2_SECRET_KEY}"
  #    region: "${S3_ACCOUNT2_REGION}"
  #    use_ssl: "${S3_ACCOUNT2_USE_SSL}"
  #    buckets: "${S3_ACCOUNT2_BUCKETS}"
  #    schedules: "${S3_ACCOUNT2_SCHEDULES}"
  #    #request_delay: 0  # optional, delay in seconds between items (default: 0)

  #- type: "mediawiki"
  #  name: "wiki1"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    host: "${MEDIAWIKI1_HOST}"
  #    path: "/w/"          # optional, default /w/
  #    scheme: "https"      # optional, default https
  #    page_limit: 500      # optional, max pages per namespace (default: unlimited)
  #    namespaces: "0,1"    # optional, comma-separated namespace IDs (default: content namespaces)
  #    filter_redirects: true  # optional, exclude redirect pages (default: true)
  #    username: "${MEDIAWIKI1_USERNAME}"  # optional, for private wikis
  #    password: "${MEDIAWIKI1_PASSWORD}"  # optional, for private wikis
  #    #verify_ssl: true     # optional, verify TLS certs (default: true)
  #    #resolve_to_ip: "10.0.0.1"  # optional, curl --resolve style IP override
  #    #user_agent: "MyBot/1.0"  # optional, override HTTP User-Agent
  #    #custom_headers:     # optional, extra HTTP headers on all API requests
  #    #  Authorization: "Bearer token"
  #    load_semantics: false  # optional, query Semantic MediaWiki properties per page (default: false)
  #    schedules: "${MEDIAWIKI1_SCHEDULES}"

  #- type: "serpapi"
  #  name: "serp_ingestion"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    api_key: "${SERPAPI_KEY}"
  #    queries: "${SERPAPI_QUERIES}"
  #    schedules: "${SERPAPI_SCHEDULES}"
  #    #request_delay: 0  # optional, delay in seconds between items (default: 0)

  #- type: "web"
  #  name: "web1"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    # urls mode: scrape a fixed list of URLs (mutually exclusive with sitemap_url)
  #    urls:
  #      - https://example.com/page1
  #      - https://example.com/page2
  #    html_to_text: true   # optional, default true
  #    schedules: "${WEB1_SCHEDULES}"

  #- type: "web"
  #  name: "web2"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    # sitemap mode: discover URLs from a sitemap (mutually exclusive with urls)
  #    sitemap_url: https://example.com/sitemap.xml
  #    include_prefix: "/wiki/"   # optional: only ingest URLs containing this string
  #    html_to_text: true         # optional, default true
  #    schedules: "${WEB2_SCHEDULES}"

  #- type: "slack"
  #  name: "slack1"
  #  config:
  #    token: "${SLACK1_TOKEN}"
  #    channel_ids: "${SLACK1_CHANNEL_IDS}"           # comma-separated channel IDs (mutually exclusive with channel_patterns)
  #    # channel_patterns: "${SLACK1_CHANNEL_PATTERNS}" # comma-separated names/regex (mutually exclusive with channel_ids)
  #    # channel_types: "public_channel,private_channel" # optional, default public_channel,private_channel
  #    # earliest_date: "2024-01-01"                  # optional
  #    # latest_date: "2025-01-01"                    # optional, requires earliest_date
  #    schedules: "${SLACK1_SCHEDULES}"

  #- type: "jira"
  #  name: "jira1"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    server_url: "${JIRA1_SERVER_URL}"
  #    # auth_type: "basic" requires email + api_token (Jira Cloud)
  #    # auth_type: "token" requires only api_token (Jira Server / Data Center PAT)
  #    auth_type: "basic"
  #    email: "${JIRA1_EMAIL}"
  #    api_token: "${JIRA1_API_TOKEN}"
  #    jql: "${JIRA1_JQL}"
  #    max_results: 50           # optional, default 50
  #    schedules: "${JIRA1_SCHEDULES}"
  #    # Bonus: load top N comments per issue
  #    load_comments: false      # optional, default false
  #    max_comments: 10          # optional, default 10
  #    #request_delay: 0  # optional, delay in seconds between items (default: 0)

  #- type: "pipedrive"
  #  name: "pipedrive1"
  #  #enabled: true  # optional; omit or set true to enable, false to skip entirely
  #  config:
  #    api_token: "${PIPEDRIVE1_API_TOKEN}"
  #    schedules: "${PIPEDRIVE1_SCHEDULES}"
  #    # Optional: which entity types to load (default: all)
  #    # load_types:
  #    #   - activities
  #    #   - deals
  #    #   - notes
  #    #   - organizations
  #    #   - persons
  #    #   - products
  #    #   - projects
  #    #   - leads
  #    #   - tasks
  #    #   - mails
  #    # max_items: 500          # optional, global per-entity limit (default: unlimited)
  #    # request_delay: 1        # optional, seconds between API requests (default: 0)
  #    # max_retries: 3          # optional, retries on failure (default: 1)
  #    # filter_deals_updated_since: "2025-01-01"   # optional ISO date
  #    # filter_activities_updated_since: "2025-01-01"
  #    # filter_deals_stages_ids:
  #    #   - 1
  #    # filter_deals_filter_id: "xxx"
  #    # filter_organizations_filter_id: "xxx"
  #    # filter_persons_filter_id: "xxx"
  #    # filter_mail_folders:
  #    #   - inbox

  #- type: "imap"
  #  name: "imap1"
  #  config:
  #    host: "${IMAP1_HOST}"
  #    port: 993                     # optional, default 993 (IMAPS), or 143 when use_starttls is set
  #    username: "${IMAP1_USERNAME}"
  #    password: "${IMAP1_PASSWORD}" # app-specific password for Gmail
  #    # mailboxes: "${IMAP1_MAILBOXES}" # optional, comma-separated; omit (or leave unset) to ingest all mailboxes
  #    # since: "2024-01-01"             # optional, only ingest messages on or after this date (YYYY-MM-DD)
  #    # use_starttls: false             # optional, default false; STARTTLS instead of implicit TLS
  #    schedules: "${IMAP1_SCHEDULES}"
  #    #request_delay: 0  # optional, delay in seconds between items (default: 0)
  #- type: "sharepoint"
  #  name: "sharepoint1"
  #  config:
  #    client_id: "${SHAREPOINT1_CLIENT_ID}"
  #    client_secret: "${SHAREPOINT1_CLIENT_SECRET}"
  #    tenant_id: "${SHAREPOINT1_TENANT_ID}"
  #    # sharepoint_site_id can be provided instead of sharepoint_site_name
  #    sharepoint_site_name: "MySite"
  #    # sharepoint_host_name and sharepoint_relative_url are passed through to
  #    # the reader but do NOT replace sharepoint_site_name / sharepoint_site_id
  #    # for site lookup. At least one of site_name or site_id is required.
  #    # sharepoint_host_name: "[your-tenant].sharepoint.com"
  #    # sharepoint_relative_url: "sites/YourSiteName"
  #    # sharepoint_folder_id can be provided instead of sharepoint_folder_path
  #    sharepoint_folder_path: "Documents/Reports"
  #    sharepoint_type: "file"  # "file" (default) or "page"
  #    recursive: true
  #    schedules: "${SHAREPOINT1_SCHEDULES}"

  #- type: "onedrive"
  #  name: "onedrive1"
  #  config:
  #    client_id: "${ONEDRIVE1_CLIENT_ID}"
  #    client_secret: "${ONEDRIVE1_CLIENT_SECRET}"
  #    tenant_id: "${ONEDRIVE1_TENANT_ID}"
  #    userprincipalname: "${ONEDRIVE1_USER_PRINCIPAL_NAME}"
  #    folder_path: "Documents/Reports"  # optional: relative folder path
  #    folder_id:                        # optional: OneDrive folder ID
  #    file_ids:                         # optional: comma-separated file IDs
  #    file_paths:                       # optional: comma-separated file paths
  #    mime_types:                       # optional: comma-separated MIME types to filter
  #    recursive: true                   # optional, default true
  #    max_file_size_mb: 50              # optional, default 50; files larger than this are skipped
  #    schedules: "${ONEDRIVE1_SCHEDULES}"

  #- type: "github"
  #  name: "github1"
  #  config:
  #    # Auth — use one of: personal_token OR github_app_id + github_app_installation_id + github_app_private_key
  #    personal_token: "${GITHUB1_PERSONAL_TOKEN}"
  #    #github_app_id: "${GITHUB1_APP_ID}"
  #    #github_app_installation_id: "${GITHUB1_APP_INSTALLATION_ID}"
  #    #github_app_private_key: "${GITHUB1_APP_PRIVATE_KEY}"
  #    owner: "${GITHUB1_OWNER}"
  #    repo: "${GITHUB1_REPO}"
  #    branch: "main"            # optional, default "main" (mutually exclusive with commit_sha)
  #    #commit_sha: ""           # optional (mutually exclusive with branch)
  #    include_extensions: "md,py"  # optional, comma-separated (mutually exclusive with exclude_extensions)
  #    #exclude_extensions: ""   # optional (mutually exclusive with include_extensions)
  #    #include_directories: ""  # optional, comma-separated (mutually exclusive with exclude_directories)
  #    #exclude_directories: ""  # optional (mutually exclusive with include_directories)
  #    include_issues: false     # optional, default false
  #    #include_issues_labels: ""   # optional, comma-separated (mutually exclusive with exclude_issues_labels)
  #    #exclude_issues_labels: ""   # optional (mutually exclusive with include_issues_labels)
  #    concurrent_requests: 5    # optional, default 5
  #    schedules: "${GITHUB1_SCHEDULES}"

  #- type: "notion"
  #  name: "notion1"
  #  config:
  #    integration_token: "${NOTION1_INTEGRATION_TOKEN}"  # Notion integration token (secret_...)
  #    page_ids: "${NOTION1_PAGE_IDS}"                    # optional: comma-separated page IDs
  #    database_ids: "${NOTION1_DATABASE_IDS}"            # optional: comma-separated database IDs
  #    request_delay: 0.3                                 # optional: delay between API calls in seconds
  #    schedules: "${NOTION1_SCHEDULES}"

embedding:
  provider: openrouter
  model_config: sentence-transformers/all-mpnet-base-v2
  embedding_dim: 768

inference:
  provider: openrouter
  model_config: openai/gpt-oss-120b:free

vector_store:
  table_name: embeddings
  hybrid_search: true
  chunk_size: 512
  chunk_overlap: 50
  hnsw:
    hnsw_m: 16
    hnsw_ef_construction: 64
    hnsw_ef_search: 40
    hnsw_dist_method: vector_cosine_ops