feat: add automatic text chunking with aggregate embeddings

Add automatic text chunking for documents exceeding embedding model token limits:

- Create chunker package with configurable chunking strategies
  - FixedOverlapChunker: splits text with configurable size and overlap
  - Uses tiktoken for accurate token counting (cl100k_base encoding)
  - Default: 512 token chunks with 50 token overlap, 8191 max tokens

- Implement batch embedding support
  - Add BatchEmbeddingProvider interface for efficient multi-text embedding
  - Implement EmbedBatch in OpenAI provider (up to 2048 texts per request)
  - Automatic fallback to individual embeddings if batch not supported

- Aggregate embedding approach
  - Chunk text when exceeding token limits
  - Embed all chunks using batch API for performance
  - Average chunk embeddings into single aggregate embedding
  - Store as single entry (no derived keys needed)

- Convert entire codebase from float32 to float64
  - Use OpenAI's native float64 format (eliminates conversions)
  - Update all similarity functions and backends
  - Better precision for similarity calculations

- Configuration options
  - Chunking enabled by default with sensible defaults
  - WithChunking() to customize chunk size/overlap/strategy
  - WithoutChunking() to disable if not needed
  - Zero-config works out of the box

- Add comprehensive tests
  - Chunker package tests (validation, chunking logic, token counting)
  - Cache integration tests (aggregate embeddings, enable/disable)
  - All existing tests pass

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: botirkhaltaev

backends/inmemory/fifo.go

@@ -11,7 +11,7 @@ import (
 type FIFOBackend[K comparable, V any] struct {
 	mu       *sync.RWMutex
 	entries  map[K]types.Entry[V]
-	index    map[K][]float32
+	index    map[K][]float64
 	queue    []K
 	capacity int
 }
@@ -21,7 +21,7 @@ func NewFIFOBackend[K comparable, V any](config types.BackendConfig) (*FIFOBacke
 	return &FIFOBackend[K, V]{
 		mu:       &sync.RWMutex{},
 		entries:  make(map[K]types.Entry[V]),
-		index:    make(map[K][]float32),
+		index:    make(map[K][]float64),
 		queue:    make([]K, 0, config.Capacity),
 		capacity: config.Capacity,
 	}, nil
@@ -102,7 +102,7 @@ func (b *FIFOBackend[K, V]) Flush(ctx context.Context) error {
 	defer b.mu.Unlock()
 
 	b.entries = make(map[K]types.Entry[V])
-	b.index = make(map[K][]float32)
+	b.index = make(map[K][]float64)
 	b.queue = make([]K, 0, b.capacity)
 	return nil
 }
@@ -128,7 +128,7 @@ func (b *FIFOBackend[K, V]) Keys(ctx context.Context) ([]K, error) {
 }
 
 // GetEmbedding retrieves just the embedding for a key
-func (b *FIFOBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float32, bool, error) {
+func (b *FIFOBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float64, bool, error) {
 	b.mu.RLock()
 	defer b.mu.RUnlock()
 

backends/inmemory/lfu.go

@@ -17,7 +17,7 @@ type LFUEntry[V any] struct {
 type LFUBackend[K comparable, V any] struct {
 	mu       *sync.RWMutex
 	entries  map[K]*LFUEntry[V]
-	index    map[K][]float32
+	index    map[K][]float64
 	capacity int
 }
 
@@ -26,7 +26,7 @@ func NewLFUBackend[K comparable, V any](config types.BackendConfig) (*LFUBackend
 	return &LFUBackend[K, V]{
 		mu:       &sync.RWMutex{},
 		entries:  make(map[K]*LFUEntry[V]),
-		index:    make(map[K][]float32),
+		index:    make(map[K][]float64),
 		capacity: config.Capacity,
 	}, nil
 }
@@ -111,7 +111,7 @@ func (b *LFUBackend[K, V]) Flush(ctx context.Context) error {
 	defer b.mu.Unlock()
 
 	b.entries = make(map[K]*LFUEntry[V])
-	b.index = make(map[K][]float32)
+	b.index = make(map[K][]float64)
 	return nil
 }
 
@@ -136,7 +136,7 @@ func (b *LFUBackend[K, V]) Keys(ctx context.Context) ([]K, error) {
 }
 
 // GetEmbedding retrieves just the embedding for a key (without incrementing frequency)
-func (b *LFUBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float32, bool, error) {
+func (b *LFUBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float64, bool, error) {
 	b.mu.RLock()
 	defer b.mu.RUnlock()
 

backends/inmemory/lru.go

@@ -12,7 +12,7 @@ import (
 type LRUBackend[K comparable, V any] struct {
 	mu    *sync.RWMutex
 	cache *lru.Cache[K, types.Entry[V]]
-	index map[K][]float32
+	index map[K][]float64
 }
 
 // NewLRUBackend creates a new LRU backend
@@ -25,7 +25,7 @@ func NewLRUBackend[K comparable, V any](config types.BackendConfig) (*LRUBackend
 	return &LRUBackend[K, V]{
 		mu:    &sync.RWMutex{},
 		cache: lruCache,
-		index: make(map[K][]float32),
+		index: make(map[K][]float64),
 	}, nil
 }
 
@@ -74,7 +74,7 @@ func (b *LRUBackend[K, V]) Flush(ctx context.Context) error {
 	defer b.mu.Unlock()
 
 	b.cache.Purge()
-	b.index = make(map[K][]float32)
+	b.index = make(map[K][]float64)
 	return nil
 }
 
@@ -93,7 +93,7 @@ func (b *LRUBackend[K, V]) Keys(ctx context.Context) ([]K, error) {
 
 	// Clean up stale index entries and collect valid keys
 	keys := make([]K, 0, b.cache.Len())
-	validIndex := make(map[K][]float32)
+	validIndex := make(map[K][]float64)
 
 	for key, embedding := range b.index {
 		if b.cache.Contains(key) {
@@ -108,7 +108,7 @@ func (b *LRUBackend[K, V]) Keys(ctx context.Context) ([]K, error) {
 }
 
 // GetEmbedding retrieves just the embedding for a key
-func (b *LRUBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float32, bool, error) {
+func (b *LRUBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float64, bool, error) {
 	b.mu.RLock()
 	embedding, hasEmbedding := b.index[key]
 	cacheHasKey := b.cache.Contains(key)

backends/remote/redis.go

@@ -193,23 +193,14 @@ func floatsToBytes(fs []float64) []byte {
 	return buf
 }
 
-// float32ToFloat64 converts float32 slice to float64 slice
-func float32ToFloat64(fs []float32) []float64 {
-	result := make([]float64, len(fs))
-	for i, f := range fs {
-		result[i] = float64(f)
-	}
-	return result
-}
-
 // Set stores an entry in Redis using JSON.SET
 func (b *RedisBackend[K, V]) Set(ctx context.Context, key K, entry types.Entry[V]) error {
 	redisKey := b.keyString(key)
 
 	doc := redisDocument[V]{
 		Key:       fmt.Sprintf("%v", key),
 		Value:     entry.Value,
-		Embedding: float32ToFloat64(entry.Embedding),
+		Embedding: entry.Embedding,
 		Timestamp: time.Now().Unix(),
 	}
 
@@ -245,14 +236,8 @@ func (b *RedisBackend[K, V]) Get(ctx context.Context, key K) (types.Entry[V], bo
 
 	doc := docs[0]
 
-	// Convert float64 back to float32
-	embedding := make([]float32, len(doc.Embedding))
-	for i, f := range doc.Embedding {
-		embedding[i] = float32(f)
-	}
-
 	entry := types.Entry[V]{
-		Embedding: embedding,
+		Embedding: doc.Embedding,
 		Value:     doc.Value,
 	}
 
@@ -371,7 +356,7 @@ func (b *RedisBackend[K, V]) Keys(ctx context.Context) ([]K, error) {
 }
 
 // GetEmbedding retrieves just the embedding for a key using JSON.GET
-func (b *RedisBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float32, bool, error) {
+func (b *RedisBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float64, bool, error) {
 	redisKey := b.keyString(key)
 
 	result, err := b.client.JSONGet(ctx, redisKey, "$.embedding").Result()
@@ -391,20 +376,13 @@ func (b *RedisBackend[K, V]) GetEmbedding(ctx context.Context, key K) ([]float32
 		return nil, false, nil
 	}
 
-	// Convert float64 back to float32
-	embedding := make([]float32, len(embeddings[0]))
-	for i, f := range embeddings[0] {
-		embedding[i] = float32(f)
-	}
-
-	return embedding, true, nil
+	return embeddings[0], true, nil
 }
 
 // VectorSearch performs vector similarity search using Redis FT.SEARCH
-func (b *RedisBackend[K, V]) VectorSearch(ctx context.Context, queryEmbedding []float32, threshold float32, limit int) ([]K, error) {
+func (b *RedisBackend[K, V]) VectorSearch(ctx context.Context, queryEmbedding []float64, threshold float64, limit int) ([]K, error) {
 	// Convert embedding to bytes for search
-	embedding64 := float32ToFloat64(queryEmbedding)
-	embeddingBytes := floatsToBytes(embedding64)
+	embeddingBytes := floatsToBytes(queryEmbedding)
 
 	// Perform vector search
 	query := fmt.Sprintf("*=>[KNN %d @embedding $vec AS vector_distance]", limit)
@@ -440,7 +418,7 @@ func (b *RedisBackend[K, V]) VectorSearch(ctx context.Context, queryEmbedding []
 		similarity := 1.0 - distance
 
 		// Check if similarity meets threshold
-		if float32(similarity) >= threshold {
+		if similarity >= threshold {
 			keyStr, ok := doc.Fields["key"]
 			if !ok {
 				continue
@@ -542,13 +520,9 @@ func (b *RedisBackend[K, V]) GetBatchAsync(ctx context.Context, keys []K) <-chan
 			}
 
 			doc := docs[0]
-			embedding := make([]float32, len(doc.Embedding))
-			for i, f := range doc.Embedding {
-				embedding[i] = float32(f)
-			}
 
 			entries[key] = types.Entry[V]{
-				Embedding: embedding,
+				Embedding: doc.Embedding,
 				Value:     doc.Value,
 			}
 		}