Skip to content

Commit c9d4381

Browse files
rex993rex993
authored
Upgrade PostgreSQL to v16 with automatic ColPali function initialization and fix Docker logging visibility │ (#190)
1 parent 82ca1f2 commit c9d4381

6 files changed

Lines changed: 92 additions & 38 deletions

File tree

‎core/api.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from core.app_factory import lifespan
1919
from core.auth_utils import verify_token
2020
from core.config import get_settings
21+
from core.logging_config import setup_logging
2122
from core.database.postgres_database import PostgresDatabase
2223
from core.dependencies import get_redis_pool
2324
from core.limits_utils import check_and_increment_limits
@@ -42,6 +43,9 @@
4243
from core.services.telemetry import TelemetryService
4344
from core.services_init import document_service
4445

46+
# Set up logging configuration for Docker environment
47+
setup_logging()
48+
4549
# Initialize FastAPI app
4650
logger = logging.getLogger(__name__)
4751

‎core/app_factory.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,21 @@ async def lifespan(app_instance: FastAPI):
5353
exc_info=True,
5454
)
5555

56+
# Initialize ColPali vector store if it exists
57+
# Note: max_sim function creation happens in MultiVectorStore.initialize()
58+
logger.info("Lifespan: Initializing ColPali Vector Store…")
59+
try:
60+
from core.services_init import colpali_vector_store
61+
if colpali_vector_store and hasattr(colpali_vector_store, "initialize"):
62+
colpali_vector_store.initialize() # This is sync method
63+
logger.info("Lifespan: ColPali Vector Store initialization successful (or not applicable).")
64+
except Exception as exc: # noqa: BLE001
65+
logger.error(
66+
"Lifespan: CRITICAL - Failed to initialize ColPali Vector Store: %s",
67+
exc,
68+
exc_info=True,
69+
)
70+
5671
logger.info("Lifespan: Attempting to initialize Redis connection pool…")
5772
global _global_redis_pool # pylint: disable=global-statement
5873
try:

‎core/services_init.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
logger = logging.getLogger(__name__)
3535

36+
3637
# ---------------------------------------------------------------------------
3738
# Global settings
3839
# ---------------------------------------------------------------------------
@@ -127,10 +128,10 @@
127128
colpali_vector_store = None
128129
case "local":
129130
colpali_embedding_model = ColpaliEmbeddingModel()
130-
colpali_vector_store = MultiVectorStore(uri=settings.POSTGRES_URI)
131+
colpali_vector_store = MultiVectorStore(uri=settings.POSTGRES_URI, auto_initialize=False)
131132
case "api":
132133
colpali_embedding_model = ColpaliApiEmbeddingModel()
133-
colpali_vector_store = MultiVectorStore(uri=settings.POSTGRES_URI)
134+
colpali_vector_store = MultiVectorStore(uri=settings.POSTGRES_URI, auto_initialize=False)
134135
case _:
135136
raise ValueError(f"Unsupported COLPALI_MODE: {settings.COLPALI_MODE}")
136137

‎core/vector_store/multi_vector_store.py‎

Lines changed: 47 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -186,45 +186,59 @@ def initialize(self):
186186
# Log index creation failure but continue
187187
logger.warning(f"Failed to create index: {str(e)}")
188188

189+
# Create max_sim function for multi-vector similarity search
190+
# This function is specific to multi-vector operations and belongs here
189191
try:
190-
# First, try to drop the existing function if it exists
191192
with self.get_connection() as conn:
192-
conn.execute(
193+
exists_check = conn.execute(
193194
"""
194-
DROP FUNCTION IF EXISTS max_sim(bit[], bit[])
195+
SELECT EXISTS (
196+
SELECT 1 FROM pg_proc
197+
WHERE proname = 'max_sim'
198+
AND pg_get_function_arguments(oid) = 'document bit[], query bit[]'
199+
)
195200
"""
196-
)
197-
logger.info("Dropped existing max_sim function")
198-
199-
# Create max_sim function
200-
conn.execute(
201+
).fetchone()[0]
202+
203+
if not exists_check:
204+
logger.info("Creating max_sim function for multi-vector similarity search")
205+
conn.execute(
206+
"""
207+
CREATE OR REPLACE FUNCTION public.max_sim(document bit[], query bit[])
208+
RETURNS double precision
209+
LANGUAGE SQL
210+
IMMUTABLE
211+
PARALLEL SAFE
212+
AS $$
213+
WITH queries AS (
214+
SELECT row_number() OVER () AS query_number, *
215+
FROM (SELECT unnest(query) AS query) AS foo
216+
),
217+
documents AS (
218+
SELECT unnest(document) AS document
219+
),
220+
similarities AS (
221+
SELECT
222+
query_number,
223+
1.0 - (bit_count(document # query)::float /
224+
greatest(bit_length(query), 1)::float) AS similarity
225+
FROM queries CROSS JOIN documents
226+
),
227+
max_similarities AS (
228+
SELECT MAX(similarity) AS max_similarity FROM similarities GROUP BY query_number
229+
)
230+
SELECT COALESCE(SUM(max_similarity), 0.0) FROM max_similarities
231+
$$
201232
"""
202-
CREATE OR REPLACE FUNCTION max_sim(document bit[], query bit[]) RETURNS double precision AS $$
203-
WITH queries AS (
204-
SELECT row_number() OVER () AS query_number, *
205-
FROM (SELECT unnest(query) AS query) AS foo
206-
),
207-
documents AS (
208-
SELECT unnest(document) AS document
209-
),
210-
similarities AS (
211-
SELECT
212-
query_number,
213-
1.0 - (bit_count(document # query)::float /
214-
greatest(bit_length(query), 1)::float) AS similarity
215-
FROM queries CROSS JOIN documents
216-
),
217-
max_similarities AS (
218-
SELECT MAX(similarity) AS max_similarity FROM similarities GROUP BY query_number
219-
)
220-
SELECT SUM(max_similarity) FROM max_similarities
221-
$$ LANGUAGE SQL
222-
"""
223-
)
224-
logger.info("Created max_sim function successfully")
233+
)
234+
conn.commit()
235+
logger.info("Created max_sim function successfully")
236+
else:
237+
logger.debug("max_sim function already exists")
238+
225239
except Exception as e:
226-
logger.error(f"Error creating max_sim function: {str(e)}")
227-
# Continue even if function creation fails - it might already exist and be usable
240+
logger.error(f"Error creating or checking max_sim function: {str(e)}")
241+
# Continue - we'll get a runtime error if the function is actually missing
228242

229243
logger.info("MultiVectorStore initialized successfully")
230244
return True
@@ -279,7 +293,6 @@ async def query_similar(
279293
doc_ids: Optional[List[str]] = None,
280294
) -> List[DocumentChunk]:
281295
"""Find similar chunks using the max_sim function for multi-vectors."""
282-
# try:
283296
# Convert query embeddings to binary format
284297
binary_query_embeddings = self._binary_quantize(query_embedding)
285298

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
x-ollama-check: &ollama-check
32
# This command reads the toml file and checks if any provider is set to "ollama"
43
command: >
@@ -139,3 +138,4 @@ volumes:
139138
ollama_data:
140139
huggingface_cache:
141140
redis_data:
141+

‎postgres.dockerfile‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,22 @@
1-
FROM ankane/pgvector
1+
FROM postgres:16-alpine
2+
3+
# Install build dependencies
4+
RUN apk add --no-cache \
5+
git \
6+
build-base \
7+
clang19 \
8+
llvm19 \
9+
postgresql-dev
10+
11+
# Clone and build pgvector
12+
RUN git clone --branch v0.8.0 https://github.com/pgvector/pgvector.git \
13+
&& cd pgvector \
14+
&& make OPTFLAGS="" \
15+
&& make install
16+
17+
# Cleanup
18+
RUN apk del git build-base clang19 llvm19 postgresql-dev \
19+
&& rm -rf /pgvector
20+
21+
# Copy data dump
22+
COPY dump.sql /tmp/dump.sql

0 commit comments

Comments
 (0)