Iris: Synchronize lecture metadata and visibility (#708)

Author: bassner

iris/src/iris/domain/ingestion/lecture_metadata_update_dto.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class LectureUnitMetadataUpdateDTO(BaseModel):
+    """Lightweight lecture-unit metadata payload sent by Artemis."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    lecture_unit_id: int = Field(alias="lectureUnitId")
+    lecture_unit_name: str = Field(default="", alias="lectureUnitName")
+    lecture_unit_link: str = Field(default="", alias="lectureUnitLink")
+    lecture_id: int = Field(alias="lectureId")
+    lecture_name: str = Field(default="", alias="lectureName")
+    course_id: int = Field(alias="courseId")
+    course_name: str = Field(default="", alias="courseName")
+    course_description: str = Field(default="", alias="courseDescription")
+    video_link: str = Field(default="", alias="videoLink")
+    base_url: str = Field(alias="baseUrl")

iris/src/iris/domain/ingestion/lecture_visibility_update_dto.py

@@ -0,0 +1,23 @@
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field
+
+
+class SlideVisibilityDTO(BaseModel):
+    """Visibility state for one PDF slide."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    slide_number: int = Field(alias="slideNumber", ge=1)
+    hidden_until: AwareDatetime | None = Field(default=None, alias="hiddenUntil")
+
+
+class LectureUnitVisibilityUpdateDTO(BaseModel):
+    """Lightweight lecture-unit visibility payload sent by Artemis."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    lecture_unit_id: int = Field(alias="lectureUnitId")
+    lecture_id: int = Field(alias="lectureId")
+    course_id: int = Field(alias="courseId")
+    base_url: str = Field(alias="baseUrl")
+    release_date: AwareDatetime | None = Field(default=None, alias="releaseDate")
+    slides: list[SlideVisibilityDTO] = Field(default_factory=list)

iris/src/iris/pipeline/chat/mcq_chat_mixin.py

@@ -15,6 +15,10 @@
 from iris.domain.status.activity_dto import ActivityKind
 from iris.pipeline.shared.mcq_generation_pipeline import McqGenerationPipeline
 from iris.retrieval.lecture.lecture_retrieval_utils import should_allow_lecture_tool
+from iris.retrieval.lecture.lecture_visibility import (
+    is_slide_visible,
+    is_unit_released,
+)
 from iris.vector_database.lecture_unit_page_chunk_schema import (
     LectureUnitPageChunkSchema,
 )
@@ -23,6 +27,10 @@
 logger = get_logger(__name__)
 
 _MAX_MCQ_COUNT = 10
+_MCQ_CHUNK_PAGE_SIZE = 100
+_MAX_MCQ_CANDIDATES = 10_000
+_MAX_MCQ_VISIBLE_CHUNKS = 50
+_MAX_MCQ_CONTENT_CHARS = 40_000
 
 
 def detect_mcq_intent(user_message: str) -> tuple[bool, int]:
@@ -74,6 +82,7 @@ def detect_mcq_intent(user_message: str) -> tuple[bool, int]:
 def retrieve_lecture_content_for_mcq(
     db: Any,
     course_id: int,
+    base_url: str,
     lecture_id: Optional[int] = None,
     allow_lecture_tool: Optional[bool] = None,
 ) -> tuple[Optional[str], list[dict]]:
@@ -85,6 +94,7 @@ def retrieve_lecture_content_for_mcq(
     Args:
         db: The Weaviate database client wrapper.
         course_id: ID of the course.
+        base_url: Artemis instance URL used to isolate colliding local IDs.
         lecture_id: Optional lecture ID to narrow results.
         allow_lecture_tool: Pre-computed lecture availability flag (e.g. from
             ``prepare_state``). Pass it to skip the redundant Weaviate check.
@@ -100,74 +110,118 @@ def retrieve_lecture_content_for_mcq(
         chunk_filter = Filter.by_property(
             LectureUnitPageChunkSchema.COURSE_ID.value
         ).equal(course_id)
+        chunk_filter &= Filter.by_property(
+            LectureUnitPageChunkSchema.BASE_URL.value
+        ).equal(base_url)
 
         if lecture_id is not None:
             chunk_filter &= Filter.by_property(
                 LectureUnitPageChunkSchema.LECTURE_ID.value
             ).equal(lecture_id)
 
-        chunks = db.lectures.query.fetch_objects(
-            filters=chunk_filter,
-            return_properties=[
-                LectureUnitPageChunkSchema.LECTURE_UNIT_ID.value,
-                LectureUnitPageChunkSchema.LECTURE_ID.value,
-                LectureUnitPageChunkSchema.PAGE_NUMBER.value,
-                LectureUnitPageChunkSchema.PAGE_TEXT_CONTENT.value,
-            ],
-        )
-
-        if not chunks.objects:
-            return None, []
-
         unit_filter = Filter.by_property(LectureUnitSchema.COURSE_ID.value).equal(
             course_id
         )
+        unit_filter &= Filter.by_property(LectureUnitSchema.BASE_URL.value).equal(
+            base_url
+        )
         unit_results = db.lecture_units.query.fetch_objects(
             filters=unit_filter,
+            limit=10_000,
             return_properties=[
                 LectureUnitSchema.LECTURE_UNIT_ID.value,
                 LectureUnitSchema.LECTURE_NAME.value,
                 LectureUnitSchema.LECTURE_UNIT_NAME.value,
+                LectureUnitSchema.RELEASE_DATE.value,
+                LectureUnitSchema.BASE_URL.value,
             ],
         )
-        unit_name_map: dict[int, dict] = {}
+        unit_name_map: dict[tuple[str, int], dict] = {}
         for obj in unit_results.objects:
             props = obj.properties
             lu_id = props.get(LectureUnitSchema.LECTURE_UNIT_ID.value)
-            if lu_id is not None:
-                unit_name_map[lu_id] = {
+            unit_base_url = props.get(LectureUnitSchema.BASE_URL.value)
+            if lu_id is not None and unit_base_url == base_url:
+                unit_name_map[(unit_base_url, lu_id)] = {
                     "lecture_name": props.get(LectureUnitSchema.LECTURE_NAME.value, ""),
                     "unit_name": props.get(
                         LectureUnitSchema.LECTURE_UNIT_NAME.value, ""
                     ),
+                    "released": is_unit_released(props),
                 }
 
-        content = ""
+        content_parts: list[str] = []
+        content_length = 0
+        visible_chunk_count = 0
         units_data: dict[int, dict] = {}
-        for obj in chunks.objects:
-            props = obj.properties
-            lu_id = props.get(LectureUnitPageChunkSchema.LECTURE_UNIT_ID.value)
-            page = props.get(LectureUnitPageChunkSchema.PAGE_NUMBER.value, 1)
-            text = props.get(LectureUnitPageChunkSchema.PAGE_TEXT_CONTENT.value, "")
-            names = unit_name_map.get(lu_id, {})
-            lecture_name = names.get("lecture_name", "")
-            unit_name = names.get("unit_name", "")
-
-            if text:
-                content += (
+        offset = 0
+        while (
+            offset < _MAX_MCQ_CANDIDATES
+            and visible_chunk_count < _MAX_MCQ_VISIBLE_CHUNKS
+            and content_length < _MAX_MCQ_CONTENT_CHARS
+        ):
+            page_limit = min(_MCQ_CHUNK_PAGE_SIZE, _MAX_MCQ_CANDIDATES - offset)
+            chunks = db.lectures.query.fetch_objects(
+                filters=chunk_filter,
+                limit=page_limit,
+                offset=offset,
+                return_properties=[
+                    LectureUnitPageChunkSchema.LECTURE_UNIT_ID.value,
+                    LectureUnitPageChunkSchema.LECTURE_ID.value,
+                    LectureUnitPageChunkSchema.PAGE_NUMBER.value,
+                    LectureUnitPageChunkSchema.PAGE_TEXT_CONTENT.value,
+                    LectureUnitPageChunkSchema.HIDDEN_UNTIL.value,
+                    LectureUnitPageChunkSchema.BASE_URL.value,
+                ],
+            )
+            if not chunks.objects:
+                break
+
+            for obj in chunks.objects:
+                props = obj.properties
+                lu_id = props.get(LectureUnitPageChunkSchema.LECTURE_UNIT_ID.value)
+                chunk_base_url = props.get(LectureUnitPageChunkSchema.BASE_URL.value)
+                if chunk_base_url != base_url:
+                    continue
+                page = props.get(LectureUnitPageChunkSchema.PAGE_NUMBER.value, 1)
+                text = props.get(LectureUnitPageChunkSchema.PAGE_TEXT_CONTENT.value, "")
+                names = unit_name_map.get((chunk_base_url, lu_id), {})
+                if (
+                    not text
+                    or not names.get("released", False)
+                    or not is_slide_visible(props)
+                ):
+                    continue
+                lecture_name = names.get("lecture_name", "")
+                unit_name = names.get("unit_name", "")
+                fragment = (
                     f"Lecture: {lecture_name}, Unit: {unit_name}, "
                     f"Page {page}\n{text}\n\n"
                 )
-
-            if lu_id is not None:
-                if lu_id not in units_data:
-                    units_data[lu_id] = {
-                        "lecture_unit_id": lu_id,
-                        "lecture_name": lecture_name,
-                        "unit_name": unit_name,
-                        "pages": set(),
-                    }
-                units_data[lu_id]["pages"].add(page)
+                remaining = _MAX_MCQ_CONTENT_CHARS - content_length
+                fragment = fragment[:remaining]
+                content_parts.append(fragment)
+                content_length += len(fragment)
+                visible_chunk_count += 1
+
+                if lu_id is not None:
+                    if lu_id not in units_data:
+                        units_data[lu_id] = {
+                            "lecture_unit_id": lu_id,
+                            "lecture_name": lecture_name,
+                            "unit_name": unit_name,
+                            "pages": set(),
+                        }
+                    units_data[lu_id]["pages"].add(page)
+                if (
+                    visible_chunk_count >= _MAX_MCQ_VISIBLE_CHUNKS
+                    or content_length >= _MAX_MCQ_CONTENT_CHARS
+                ):
+                    break
+
+            offset += len(chunks.objects)
+            if len(chunks.objects) < page_limit:
+                break
 
         lecture_units_meta = []
         for data in units_data.values():
@@ -176,6 +230,7 @@ def retrieve_lecture_content_for_mcq(
             del data["pages"]
             lecture_units_meta.append(data)
 
+        content = "".join(content_parts)
         return (content if content.strip() else None), lecture_units_meta
     except Exception as e:
         logger.warning("Failed to fetch lecture summaries for MCQ: %s", str(e))
@@ -244,10 +299,12 @@ def mcq_pre_agent_hook(
 
     user_message = get_text_of_latest_user_message(state)
     count = getattr(state, "mcq_count", 1)
+    execution_settings = getattr(state.dto, "settings", None)
 
     lecture_content, _ = retrieve_lecture_content_for_mcq(
         db,
         course_id,
+        execution_settings.artemis_base_url if execution_settings else "",
         lecture_id=lecture_id,
         allow_lecture_tool=getattr(state, "allow_lecture_tool", None),
     )

iris/src/iris/pipeline/lecture_ingestion_pipeline.py

@@ -4,6 +4,7 @@
 import re
 import tempfile
 import threading
+from datetime import datetime
 from typing import Optional
 
 import fitz
@@ -36,6 +37,10 @@
     LectureUnitPageChunkSchema,
     init_lecture_unit_page_chunk_schema,
 )
+from ..vector_database.lecture_unit_schema import (
+    LectureUnitSchema,
+    init_lecture_unit_schema,
+)
 from ..web.status import ingestion_status_callback
 from . import Pipeline
 
@@ -126,6 +131,7 @@ def create_page_data(
     course_language,
     base_url,
     display_page_number,
+    hidden_until=None,
 ):
     """
     Create and return a list of dictionnaries to be ingested in the Vector Database.
@@ -141,6 +147,7 @@ def create_page_data(
             LectureUnitPageChunkSchema.PAGE_TEXT_CONTENT.value: page_split.page_content,
             LectureUnitPageChunkSchema.BASE_URL.value: base_url,
             LectureUnitPageChunkSchema.PAGE_VERSION.value: lecture_unit_dto.attachment_version,
+            LectureUnitPageChunkSchema.HIDDEN_UNTIL.value: hidden_until,
         }
         for page_split in page_splits
     ]
@@ -176,6 +183,7 @@ def __init__(
     ):
         super().__init__(implementation_id=self.PIPELINE_ID)
         self.collection = init_lecture_unit_page_chunk_schema(client)
+        self.lecture_unit_collection = init_lecture_unit_schema(client)
         self.dto = dto
         self.callback = callback
         chat_model = variant.model("chat", local)
@@ -190,6 +198,7 @@ def __init__(
         self.pipeline = self.llm | StrOutputParser()
         self.tokens = []
         self.course_language = None
+        self._hidden_until_by_page: dict[int, object] = {}
 
     @observe(name="Lecture Unit Page Ingestion Pipeline")
     def __call__(self) -> (str, []):
@@ -212,6 +221,7 @@ def __call__(self) -> (str, []):
                 self.callback.update()
                 return self.course_language, self.tokens
             self.callback.update()
+            self._load_existing_slide_visibility()
             self.delete_lecture_unit(
                 self.dto.lecture_unit.course_id,
                 self.dto.lecture_unit.lecture_id,
@@ -283,6 +293,56 @@ def _get_page_chunk_filter(self):
         ).equal(self.dto.lecture_unit.lecture_unit_id)
         return page_chunk_filter
 
+    def _load_existing_slide_visibility(self) -> None:
+        """Preserve visibility when full ingestion replaces page chunks."""
+        units = self.lecture_unit_collection.query.fetch_objects(
+            filters=Filter.all_of(
+                [
+                    Filter.by_property(LectureUnitSchema.BASE_URL.value).equal(
+                        self.dto.settings.artemis_base_url
+                    ),
+                    Filter.by_property(LectureUnitSchema.COURSE_ID.value).equal(
+                        self.dto.lecture_unit.course_id
+                    ),
+                    Filter.by_property(LectureUnitSchema.LECTURE_ID.value).equal(
+                        self.dto.lecture_unit.lecture_id
+                    ),
+                    Filter.by_property(LectureUnitSchema.LECTURE_UNIT_ID.value).equal(
+                        self.dto.lecture_unit.lecture_unit_id
+                    ),
+                ]
+            ),
+            limit=1,
+        ).objects
+        if units:
+            serialized = units[0].properties.get(
+                LectureUnitSchema.SLIDE_VISIBILITY.value
+            )
+            if serialized is not None:
+                snapshot = json.loads(serialized)
+                self._hidden_until_by_page = {
+                    int(page_number): (
+                        datetime.fromisoformat(hidden_until)
+                        if hidden_until is not None
+                        else None
+                    )
+                    for page_number, hidden_until in snapshot.items()
+                }
+                return
+
+        chunks = self.collection.query.fetch_objects(
+            filters=self._get_page_chunk_filter(), limit=10_000
+        ).objects
+        self._hidden_until_by_page = {}
+        for chunk in chunks:
+            page_number = int(
+                chunk.properties[LectureUnitPageChunkSchema.PAGE_NUMBER.value]
+            )
+            self._hidden_until_by_page.setdefault(
+                page_number,
+                chunk.properties.get(LectureUnitPageChunkSchema.HIDDEN_UNTIL.value),
+            )
+
     def restore_display_page_numbers_from_existing_chunks(self) -> None:
         chunks = self.collection.query.fetch_objects(
             filters=self._get_page_chunk_filter(), limit=10000
@@ -375,6 +435,7 @@ def chunk_data(
                     self.course_language,
                     base_url,
                     vision_result.display_page_number,
+                    self._hidden_until_by_page.get(page_num + 1),
                 )
             )
             old_page_text = page_text