fix(multi-tenancy): address PR feedback and CI auth test failures

Fix CI auth workflow failures and address review feedback:

- Add vector_store::* routing table ABAC rules alongside existing
  sql_record::vector_stores::* rules to cover both authorization layers
- Remove stale agent_state and metadata_store configs from the CI auth
  workflow to prevent SQLAlchemy lazy-init race with the responses table
- Add embedding_model parameter to vector store access control tests
- Use per-table migration guards in kvstore-to-sql migration so a crash
  mid-migration can resume from the last incomplete table on next boot

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>

Author: franciscojavierarceo

.github/workflows/integration-responses-conversations-auth-tests.yml

@@ -121,16 +121,17 @@ jobs:
                 provider_type: remote::openai
                 config:
                   api_key: \${env.OPENAI_API_KEY}
+              - provider_id: sentence-transformers
+                provider_type: inline::sentence-transformers
+                config:
+                  trust_remote_code: true
             vector_io:
               - provider_id: faiss
                 provider_type: inline::faiss
                 config:
                   persistence:
                     namespace: vector_io::faiss
                     backend: kv_default
-                  metadata_store:
-                    table_name: vector_store_metadata
-                    backend: sql_default
             tool_runtime:
               - provider_id: file-search
                 provider_type: inline::file-search
@@ -140,9 +141,6 @@ jobs:
                 provider_type: inline::builtin
                 config:
                   persistence:
-                    agent_state:
-                      namespace: responses
-                      backend: kv_default
                     responses:
                       table_name: responses
                       backend: sql_default
@@ -169,13 +167,13 @@ jobs:
               prompts:
                 table_name: prompts
                 backend: sql_default
-              vector_stores:
-                table_name: vector_store_metadata
-                backend: sql_default
           models:
           - model_id: openai/gpt-4o
             model_type: llm
             provider_id: openai
+          - model_id: sentence-transformers/nomic-ai/nomic-embed-text-v1.5
+            model_type: embedding
+            provider_id: sentence-transformers
           server:
             port: 8321
             auth:
@@ -184,6 +182,16 @@ jobs:
                     actions: [read]
                     resource: model::*
                   description: Any authenticated user can use configured models (inference, responses)
+                - permit:
+                    actions: [create]
+                    resource: vector_store::*
+                  description: Any authenticated user can create vector stores (routing table)
+                - permit:
+                    actions: [read, update, delete]
+                    resource: vector_store::*
+                  when:
+                    - user is owner
+                  description: Users can access their own vector stores (routing table)
                 - permit:
                     actions: [create]
                     resource: sql_record::openai_files::*

src/ogx/providers/utils/memory/openai_vector_store_mixin.py

@@ -225,105 +225,115 @@ async def _migrate_kvstore_to_sql(self) -> None:
 
         sql_store = self.metadata_store.sql_store
 
-        existing = await sql_store.fetch_all(table=TABLE_VECTOR_STORES, limit=1)
-        if existing.data:
-            return
-
         stores_data = await self.kvstore.values_in_range(
             OPENAI_VECTOR_STORES_PREFIX, f"{OPENAI_VECTOR_STORES_PREFIX}\xff"
         )
         if not stores_data:
             return
 
-        logger.info("Starting KVStore to SQL migration for vector store metadata", store_count=len(stores_data))
-
         migrated_stores = 0
         migrated_files = 0
         migrated_chunks = 0
         migrated_batches = 0
 
-        for raw in stores_data:
-            info = json.loads(raw)
-            store_id = info["id"]
-            await sql_store.insert(
-                table=TABLE_VECTOR_STORES,
-                data={
-                    "id": store_id,
-                    "store_data": info,
-                    "owner_principal": "",
-                    "access_attributes": None,
-                },
-            )
-            migrated_stores += 1
-
-            file_keys = await self.kvstore.keys_in_range(
-                f"{OPENAI_VECTOR_STORES_FILES_PREFIX}{store_id}:",
-                f"{OPENAI_VECTOR_STORES_FILES_PREFIX}{store_id}:\xff",
+        # Per-table migration: each table is checked independently so a crash
+        # mid-migration doesn't skip remaining tables on the next boot.
+        existing_stores = await sql_store.fetch_all(table=TABLE_VECTOR_STORES, limit=1)
+        if not existing_stores.data:
+            logger.info(
+                "Starting KVStore to SQL migration for vector store metadata",
+                store_count=len(stores_data),
             )
-            for file_key in file_keys:
-                suffix = file_key[len(OPENAI_VECTOR_STORES_FILES_PREFIX) :]
-                file_id = suffix.split(":", 1)[1] if ":" in suffix else suffix
-                raw_file = await self.kvstore.get(file_key)
-                if not raw_file:
-                    continue
-                file_info = json.loads(raw_file)
+            for raw in stores_data:
+                info = json.loads(raw)
+                store_id = info["id"]
                 await sql_store.insert(
-                    table=TABLE_VECTOR_STORE_FILES,
+                    table=TABLE_VECTOR_STORES,
                     data={
-                        "id": f"{store_id}:{file_id}",
-                        "store_id": store_id,
-                        "file_id": file_id,
-                        "file_data": file_info,
+                        "id": store_id,
+                        "store_data": info,
                         "owner_principal": "",
                         "access_attributes": None,
                     },
                 )
-                migrated_files += 1
-
-                chunk_prefix = f"{OPENAI_VECTOR_STORES_FILES_CONTENTS_PREFIX}{store_id}:{file_id}:"
-                chunk_values = await self.kvstore.values_in_range(chunk_prefix, f"{chunk_prefix}\xff")
-                for idx, raw_chunk in enumerate(chunk_values):
-                    chunk = json.loads(raw_chunk)
+                migrated_stores += 1
+
+        existing_files = await sql_store.fetch_all(table=TABLE_VECTOR_STORE_FILES, limit=1)
+        if not existing_files.data:
+            for raw in stores_data:
+                info = json.loads(raw)
+                store_id = info["id"]
+                file_keys = await self.kvstore.keys_in_range(
+                    f"{OPENAI_VECTOR_STORES_FILES_PREFIX}{store_id}:",
+                    f"{OPENAI_VECTOR_STORES_FILES_PREFIX}{store_id}:\xff",
+                )
+                for file_key in file_keys:
+                    suffix = file_key[len(OPENAI_VECTOR_STORES_FILES_PREFIX) :]
+                    file_id = suffix.split(":", 1)[1] if ":" in suffix else suffix
+                    raw_file = await self.kvstore.get(file_key)
+                    if not raw_file:
+                        continue
+                    file_info = json.loads(raw_file)
                     await sql_store.insert(
-                        table=TABLE_VECTOR_STORE_FILE_CONTENTS,
+                        table=TABLE_VECTOR_STORE_FILES,
                         data={
-                            "id": f"{store_id}:{file_id}:{idx}",
+                            "id": f"{store_id}:{file_id}",
                             "store_id": store_id,
                             "file_id": file_id,
-                            "chunk_index": idx,
-                            "chunk_data": chunk,
+                            "file_data": file_info,
                             "owner_principal": "",
                             "access_attributes": None,
                         },
                     )
-                    migrated_chunks += 1
+                    migrated_files += 1
+
+                    chunk_prefix = f"{OPENAI_VECTOR_STORES_FILES_CONTENTS_PREFIX}{store_id}:{file_id}:"
+                    chunk_values = await self.kvstore.values_in_range(chunk_prefix, f"{chunk_prefix}\xff")
+                    for idx, raw_chunk in enumerate(chunk_values):
+                        chunk = json.loads(raw_chunk)
+                        await sql_store.insert(
+                            table=TABLE_VECTOR_STORE_FILE_CONTENTS,
+                            data={
+                                "id": f"{store_id}:{file_id}:{idx}",
+                                "store_id": store_id,
+                                "file_id": file_id,
+                                "chunk_index": idx,
+                                "chunk_data": chunk,
+                                "owner_principal": "",
+                                "access_attributes": None,
+                            },
+                        )
+                        migrated_chunks += 1
 
-        batch_data = await self.kvstore.values_in_range(
-            OPENAI_VECTOR_STORES_FILE_BATCHES_PREFIX, f"{OPENAI_VECTOR_STORES_FILE_BATCHES_PREFIX}\xff"
-        )
-        for raw_batch in batch_data:
-            batch_info = json.loads(raw_batch)
-            batch_id = batch_info["id"]
-            await sql_store.insert(
-                table=TABLE_VECTOR_STORE_FILE_BATCHES,
-                data={
-                    "id": batch_id,
-                    "store_id": batch_info.get("vector_store_id", ""),
-                    "batch_data": batch_info,
-                    "expires_at": batch_info.get("expires_at", 0),
-                    "owner_principal": "",
-                    "access_attributes": None,
-                },
+        existing_batches = await sql_store.fetch_all(table=TABLE_VECTOR_STORE_FILE_BATCHES, limit=1)
+        if not existing_batches.data:
+            batch_data = await self.kvstore.values_in_range(
+                OPENAI_VECTOR_STORES_FILE_BATCHES_PREFIX, f"{OPENAI_VECTOR_STORES_FILE_BATCHES_PREFIX}\xff"
+            )
+            for raw_batch in batch_data:
+                batch_info = json.loads(raw_batch)
+                batch_id = batch_info["id"]
+                await sql_store.insert(
+                    table=TABLE_VECTOR_STORE_FILE_BATCHES,
+                    data={
+                        "id": batch_id,
+                        "store_id": batch_info.get("vector_store_id", ""),
+                        "batch_data": batch_info,
+                        "expires_at": batch_info.get("expires_at", 0),
+                        "owner_principal": "",
+                        "access_attributes": None,
+                    },
+                )
+                migrated_batches += 1
+
+        if migrated_stores or migrated_files or migrated_chunks or migrated_batches:
+            logger.info(
+                "KVStore to SQL migration complete",
+                stores=migrated_stores,
+                files=migrated_files,
+                chunks=migrated_chunks,
+                batches=migrated_batches,
             )
-            migrated_batches += 1
-
-        logger.info(
-            "KVStore to SQL migration complete",
-            stores=migrated_stores,
-            files=migrated_files,
-            chunks=migrated_chunks,
-            batches=migrated_batches,
-        )
 
     async def _save_openai_vector_store(self, store_id: str, store_info: dict[str, Any]) -> None:
         """Save vector store metadata to persistent storage."""

tests/integration/vector_io/test_vector_stores_access_control.py

@@ -52,11 +52,14 @@ def _headers(self) -> dict[str, str]:
             "Content-Type": "application/json",
         }
 
-    def create(self, name: str) -> dict:
+    def create(self, name: str, embedding_model: str | None = None) -> dict:
+        body: dict = {"name": name}
+        if embedding_model:
+            body["embedding_model"] = embedding_model
         resp = httpx.post(
             f"{self.base_url}/v1/vector_stores",
             headers=self._headers(),
-            json={"name": name},
+            json=body,
             timeout=30.0,
         )
         resp.raise_for_status()
@@ -113,8 +116,10 @@ def bob(self, ogx_client) -> VectorStoresClient:
         token = get_auth_token("BOB_TOKEN", "token-bob")
         return VectorStoresClient(str(ogx_client.base_url), token)
 
+    EMBEDDING_MODEL = "sentence-transformers/nomic-ai/nomic-embed-text-v1.5"
+
     def _create_store(self, client: VectorStoresClient, name: str = "test-store") -> str:
-        data = client.create(name)
+        data = client.create(name, embedding_model=self.EMBEDDING_MODEL)
         return data["id"]
 
     def test_user_cannot_retrieve_other_users_vector_store(self, alice, bob, require_server):