fix(pgvector): make doc deletion query faster (#289)

CI logging for slow queries has also been enabled, not sure if we will
see that in the CI though.

Sample output for the slow deletion query where a missing index on the
`source_id` foreign key in access_list table was the culprit.
Calculated time: 3.495 + 0.310 + 0.129 = 3.934 ms
Actual time: 201177.123 ms or 201 s
```
        Query Text: DELETE FROM docs WHERE docs.source_id IN ($1::VARCHAR, $2::VARCHAR, ..., $275::VARCHAR) RETURNING docs.chunks
        Query Parameters: ...
        Delete on docs  (cost=1126.32..2018.25 rows=275 width=6) (actual time=0.192..3.495 rows=218 loops=1)
    ->  Bitmap Heap Scan on docs  (cost=1126.32..2018.25 rows=275 width=6) (actual time=0.144..0.310 rows=218 loops=1)
                Recheck Cond: ((source_id)::text = ANY ('{"files__default: 20392","files__default: 23092", ... }'::text[]))
                Heap Blocks: exact=25
                ->  Bitmap Index Scan on docs_pkey  (cost=0.00..1125.56 rows=275 width=0) (actual time=0.129..0.129 rows=218 loops=1)
                      Index Cond: ((source_id)::text = ANY ('{"files__default: 20392", ...
2026-03-19 11:28:59.760 UTC [6703] LOG:  duration: 201177.123 ms  execute <unnamed>: DELETE FROM docs WHERE docs.source_id IN ($1::VARCHAR, $2::VARCHAR, ..., $275::VARCHAR) RETURNING docs.chunks
2026-03-19 11:28:59.760 UTC [6703] DETAIL:  Parameters: $1 = 'files__default: 20392', $2 = ...
```

(put the chunking part in a different PR)

Author: kyteinsky

.github/workflows/integration-test.yml

@@ -92,6 +92,41 @@ jobs:
         options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 --name postgres --hostname postgres
 
     steps:
+      - name: Enable PostgreSQL slow query logging and auto_explain
+        run: |
+          docker exec postgres bash -c "\
+            cat >> /var/lib/postgresql/data/postgresql.conf <<EOF
+
+          # slow query logging
+          log_min_duration_statement = 20
+
+          # auto_explain for slow queries
+          session_preload_libraries = 'auto_explain'
+          auto_explain.log_min_duration = '20ms'
+          auto_explain.log_analyze = on
+          auto_explain.log_buffers = on
+          auto_explain.log_nested_statements = on
+          auto_explain.log_verbose = on
+
+          # file-based logging
+          logging_collector = on
+          log_directory = '/var/log/pg_log'
+          log_filename = 'postgresql.log'
+          log_file_mode = 0644
+          EOF"
+
+          # create the log directory and set ownership for postgres user (logging_collector needs it)
+          docker exec postgres bash -c "mkdir -p /var/log/pg_log && chown -R postgres:postgres /var/log/pg_log"
+
+          # logging_collector requires a restart to take effect
+          docker restart postgres
+          # wait for postgres to be ready again (max 60 seconds)
+          timeout 60 bash -c 'until docker exec postgres pg_isready -U root; do sleep 1; done' || { echo "Error: PostgreSQL did not become ready within 60 seconds"; docker logs postgres --tail 50; exit 1; }
+
+          # verify the config has been loaded
+          docker exec postgres psql -U root -d nextcloud -c "SHOW log_min_duration_statement;"
+          docker exec postgres psql -U root -d nextcloud -c "SHOW session_preload_libraries;"
+
       - name: Checkout server
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
@@ -378,6 +413,11 @@ jobs:
             /tmp/0_pgdump_nextcloud
             /tmp/1_pgdump_nextcloud
 
+      - name: Show PostgreSQL slow query logs
+        if: always()
+        run: |
+          docker exec postgres cat /var/log/pg_log/postgresql.log
+
       - name: Final stats log
         if: always()
         run: |

README.md

@@ -124,10 +124,10 @@ v2.1.0 introduces repair steps. These run on app startup.
 
 `repair2001_date20240412153300.py` removes the existing config.yaml in the persistent storage for the
 hardware detection to run and place a suitable config (based on accelerator detected) in its place.  
-To skip this step (or steps in the future), populate the `repair.info` file with the repair file name(s).  
+To skip this step (or steps in the future), add the repair filename(s) to `repair.info` in the persistent storage, one filename per line.  
 Use the below command inside the container or add the repair filename manually in the repair.info file inside the docker container at `/nc_app_context_chat_backend_data`
 
-`echo repair2001_date20240412153300.py > "$APP_PERSISTENT_STORAGE/repair.info"`
+`echo repair2001_date20240412153300.py >> "$APP_PERSISTENT_STORAGE/repair.info"`
 
 #### How to generate a repair step file
 `APP_VERSION` should at least be incremented at the minor level (MAJOR.MINOR.PATCH)

context_chat_backend/repair/repair5004_date20260521105831.py

@@ -0,0 +1,28 @@
+#
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+import os
+
+import sqlalchemy as sa
+
+'''
+Add an index on access_list.source_id to speed up ON DELETE CASCADE
+triggered when deleting from the docs table.
+Without this index, the CASCADE performs a sequential scan of access_list
+for each deleted doc row, causing very slow batch deletes.
+'''
+
+
+def run(_previous_version: int):
+	db_url = os.environ.get('CCB_DB_URL')
+	if not db_url:
+		print('CCB_DB_URL not set, skipping access_list index migration', flush=True)
+		return
+
+	engine = sa.create_engine(db_url)
+	with engine.connect() as conn:
+		conn.execute(sa.text(
+			'CREATE INDEX IF NOT EXISTS idx_access_list_source_id ON access_list (source_id)'
+		))
+		conn.commit()

context_chat_backend/repair/runner.py

@@ -7,6 +7,11 @@
 import re
 from importlib import import_module
 
+REPAIR_DIR = 'context_chat_backend/repair'
+VERSION_INFO_FILE = 'version.info'
+REPAIR_SKIP_FILE = 'repair.info'
+PARTIAL_REPAIR_FILE = 'partial_repair.tmp'
+
 
 def get_previous_version(version_info_path: str) -> tuple[int, bool]:
 	'''
@@ -15,8 +20,15 @@ def get_previous_version(version_info_path: str) -> tuple[int, bool]:
 	if not os.path.exists(version_info_path):
 		return (0, False)
 
-	with open(version_info_path) as f:
-		version_string = f.read().strip()
+	try:
+		with open(version_info_path) as f:
+			version_string = f.read().strip()
+	except OSError as e:
+		print(
+			f'Warning: could not read {version_info_path}, assuming no previous version was installed: {e}',
+			flush=True,
+		)
+		return (0, False)
 
 	if not version_string:
 		return (0, False)
@@ -33,25 +45,55 @@ def get_previous_version(version_info_path: str) -> tuple[int, bool]:
 	return (int(major + minor.zfill(3)), repairs_pending)
 
 
+def get_skipped_repairs(persistent_storage_path: str) -> set[str]:
+	repair_info_path = os.path.join(persistent_storage_path, REPAIR_SKIP_FILE)
+	if not os.path.exists(repair_info_path):
+		return set()
+
+	try:
+		with open(repair_info_path) as f:
+			return {line.strip() for line in f if line.strip()}
+	except OSError as e:
+		print(f'Warning: could not read {repair_info_path}, no repairs will be skipped: {e}', flush=True)
+		return set()
+
+
 def main():
 	'''
 	Run repairs that have not been run before.
 	Repair files can either have no functions or a run() function.
+	To skip a repair, add its filename to repair.info in the persistent storage.
 	'''
 	print('Running repairs...', flush=True)
 
 	persistent_storage_path = os.getenv('APP_PERSISTENT_STORAGE', 'persistent_storage')
-	version_info_path = os.path.join(persistent_storage_path, 'version.info')
+	version_info_path = os.path.join(persistent_storage_path, VERSION_INFO_FILE)
+	partial_repair_path = os.path.join(persistent_storage_path, PARTIAL_REPAIR_FILE)
 
-	all_filenames = os.listdir('context_chat_backend/repair')
-	repair_filenames = [f for f in all_filenames if f.startswith('repair') and f.endswith('.py')]
+	try:
+		all_filenames = os.listdir(REPAIR_DIR)
+	except OSError as e:
+		print(f'Error: could not list repair directory to get all the eligible repairs: {e}', flush=True)
+		raise
+	repair_filenames = sorted(f for f in all_filenames if f.startswith('repair') and f.endswith('.py'))
 
 	(previous_app_version, repairs_pending) = get_previous_version(version_info_path)
 
 	if not repairs_pending:
 		print('No repairs are required.', flush=True)
 		return
 
+	skipped_repairs = get_skipped_repairs(persistent_storage_path)
+
+	try:
+		with open(partial_repair_path) as f:
+			partial_repairs = {line.strip() for line in f if line.strip()}
+	except FileNotFoundError:
+		partial_repairs = set()
+	except OSError as e:
+		print(f'Warning: could not read {partial_repair_path}, all pending repairs will be re-run: {e}', flush=True)
+		partial_repairs = set()
+
 	for repair_filename in repair_filenames:
 		pattern = re.compile(r'^repair(\d+)_date\d+\.py$')
 		matches = pattern.match(repair_filename)
@@ -65,16 +107,50 @@ def main():
 			print(f'No repairs to run for version {introduced_version}.', flush=True)
 			continue
 
+		if repair_filename in skipped_repairs:
+			print(f'Skipping repair {repair_filename} (listed in repair.info).', flush=True)
+			continue
+
+		if repair_filename in partial_repairs:
+			print(f'Skipping repair {repair_filename} (already completed in partial run).', flush=True)
+			continue
+
 		print(f'Running repair {repair_filename}...', flush=True, end='')
 
 		mod = import_module(f'.repair.{repair_filename[:-3]}', 'context_chat_backend')
 		if hasattr(mod, 'run'):
-			mod.run(previous_app_version)
+			try:
+				mod.run(previous_app_version)
+			except Exception:
+				print(
+					'failed.\n'
+					'The app will not continue further until this repair step succeeds, '
+					'or is skipped through the method described in https://github.com/nextcloud/context_chat_backend/#repair \n'  # noqa: E501
+					'If not skipped, it will be tried again in the next app startup.',
+					flush=True,
+				)
+				raise
+
+		try:
+			with open(partial_repair_path, 'a') as f:
+				f.write(repair_filename + '\n')
+		except OSError as e:
+			print(f'Warning: could not write to {partial_repair_path}: {e}', flush=True)
 
 		print('completed.', flush=True)
 
-	with open(version_info_path, 'w') as f:
-		f.write(os.environ['APP_VERSION'] + '+')
+	try:
+		if os.path.exists(partial_repair_path):
+			os.unlink(partial_repair_path)
+	except OSError as e:
+		print(f'Warning: could not remove {partial_repair_path}: {e}', flush=True)
+
+	try:
+		with open(version_info_path, 'w') as f:
+			f.write(os.environ['APP_VERSION'] + '+')
+	except OSError as e:
+		print(f'Error: could not write {version_info_path}: {e}', flush=True)
+		return
 
 	print('Repairs completed.', flush=True)
 

context_chat_backend/vectordb/pgvector.py

@@ -97,6 +97,10 @@ class AccessListStore(Base):
 			'source_id',
 			unique=True,
 		),
+		sa.Index(
+			'idx_access_list_source_id',
+			'source_id',
+		)
 	)
 
 	@classmethod