Collection-mode review fixes: auto entries moderated, withdrawable, oracle-free

pr-review findings:
- Auto-collected submissions were NEVER moderated ('no text content' —
  false: the gallery card renders the map's name/description, which the
  scorer deliberately covers). auto_finalize now returns flipped ids and
  both callers enqueue moderation post-commit; renaming a live-referenced
  map re-schedules scoring, since the card text is live.
- Live-referenced (map_is_clone=false) auto entries flip BOTH ways:
  regressing below the submitted tier withdraws the entry — the author
  never filled a consent form, and /api/submissions has no draft_status
  filter, so one-way stranded a map its author pulled back to scratch.
  Clone-backed submissions stay one-way. Docstring corrected (it claimed
  a filter that only exists on documents/list).
- /api/submissions/flag refuses internal-portal rows: flagging them
  spammed the staff queue and leaked an existence oracle for staff-only
  galleries.
- 'excluded from ALL public reads' claims scoped honestly to LISTINGS
  (any map's metadata was always fetchable by sequential public_id).
- get_document_list uses SUBMITTED_DRAFT_STATUSES instead of restating
  it; migration index renamed to the autogenerate convention; stray
  empty backend/uv.lock deleted; auto_finalize takes the same row lock
  as finalize.
- New tests: auto entries surface publicly (the auto_public/internal
  distinction had no positive assertion), map_is_clone stays false,
  takedown never demotes the live map, create-with-metadata flips at
  creation with moderation scheduled, rename re-scores, withdrawal
  round-trip, textarea cap at both bounds, internal flag-oracle.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: nofurtherinformation

backend/app/alembic/versions/e4a7c318b9d2_collection_mode_and_custom_fields.py

@@ -85,7 +85,7 @@ def upgrade() -> None:
         schema="comments",
     )
     op.create_index(
-        "idx_form_fields_custom_portal",
+        "ix_comments_form_fields_custom_portal_id",
         "form_fields_custom",
         ["portal_id"],
         schema="comments",

backend/app/main.py

@@ -69,7 +69,6 @@
 from app.evaluation.types import MetricsEnvelope
 import app.save_share.main as save_share
 import app.submissions.main as submissions
-from app.submissions.main import auto_finalize_draft_submissions
 from app.submissions.models import (
     CollectionMode,
     FormConfig,
@@ -97,7 +96,7 @@
     AssignmentsCreate,
     NumDistrictsSetResult,
 )
-from app.save_share.models import DocumentDraftStatus
+from app.save_share.models import SUBMITTED_DRAFT_STATUSES, DocumentDraftStatus
 from pydantic_geojson import FeatureModel, PolygonModel
 from pydantic import BaseModel, ValidationError
 from pydantic_geojson._base import Coordinates
@@ -584,10 +583,14 @@ async def create_document(
         if data.portal_id is not None:
             # A creation payload can already carry a submitted-tier status
             # (e.g. copies); apply the same auto-collect flip as the
-            # metadata endpoint.
-            auto_finalize_draft_submissions(
+            # metadata endpoint, with the same post-commit moderation pass
+            # (the gallery card renders the map's name/description).
+            for flipped_id in submissions.auto_finalize_draft_submissions(
                 session, new_document.public_id, data.metadata.draft_status
-            )
+            ):
+                background_tasks.add_task(
+                    submissions.moderate_submission_by_id, flipped_id
+                )
 
     stmt = (
         select(  # type: ignore[no-matching-overload] # ty: ignore[no-matching-overload]
@@ -1464,7 +1467,10 @@ async def get_document_list(
                     col(Submission.status) == SubmissionStatus.submitted,
                     col(Submission.hidden).is_(False),
                     col(Submission.nsfw).is_(False),
-                    # Internal-mode portals never surface publicly.
+                    # Internal-mode portals never surface in tag galleries
+                    # or the public submissions list. (This is a LISTING
+                    # guarantee: any map's metadata remains fetchable by its
+                    # sequential public_id, as it always has been.)
                     col(FormConfig.collection_mode) != CollectionMode.internal,
                 )
             )
@@ -1476,10 +1482,7 @@ async def get_document_list(
         # (deliberate submissions are frozen clones at ready_to_share;
         # auto-collected ones are live maps whose status this reflects).
         if len(draft_status) == 0:
-            draft_status = [
-                DocumentDraftStatus.in_progress,
-                DocumentDraftStatus.ready_to_share,
-            ]
+            draft_status = list(SUBMITTED_DRAFT_STATUSES)
 
     if len(draft_status) > 0:
         # this is fine to keep as ->> because you're comparing to text
@@ -1758,6 +1761,7 @@ async def get_connected_component_bboxes(
 )
 async def update_districtrmap_metadata(
     metadata: DocumentMetadata,
+    background_tasks: BackgroundTasks,
     document: Document = Depends(get_document),
     session: Session = Depends(get_session),
 ):
@@ -1776,11 +1780,32 @@ async def update_districtrmap_metadata(
         )
         session.connection().execute(stmt)
         # Auto-collect portals: reaching a submitted-tier status flips this
-        # map's draft submission to submitted (live reference, no clone).
-        auto_finalize_draft_submissions(
+        # map's draft submission to submitted (live reference, no clone);
+        # regressing withdraws it.
+        flipped = submissions.auto_finalize_draft_submissions(
             session, document.public_id, merged.get("draft_status")
         )
+        # Auto entries are live references, so the rendered card text (map
+        # name/description) can change AFTER the initial score — re-score
+        # submitted live-ref entries whenever those fields are touched.
+        rescore: set[int] = set(flipped)
+        if metadata.name is not None or metadata.description is not None:
+            rescore.update(
+                session.exec(
+                    select(Submission.id).where(
+                        and_(
+                            col(Submission.map_public_id) == document.public_id,
+                            col(Submission.status) == SubmissionStatus.submitted,
+                            col(Submission.map_is_clone).is_(False),
+                        )
+                    )
+                ).all()
+            )
         session.commit()
+        for submission_id in rescore:
+            background_tasks.add_task(
+                submissions.moderate_submission_by_id, submission_id
+            )
 
     except Exception as e:
         logger.error(f"Unexpected error: {e}")

backend/app/submissions/main.py

@@ -99,36 +99,54 @@ def get_form_config(portal_id: str, session: Session) -> FormConfig:
 
 def auto_finalize_draft_submissions(
     session: Session, public_id: int | None, draft_status: str | None
-) -> int:
-    """Flip auto-mode draft submissions to submitted when their map reaches a
-    submitted-tier draft_status (in_progress / ready_to_share).
+) -> list[int]:
+    """Sync auto-mode submissions with their map's draft_status.
 
     The auto-collect contract: entries keep their LIVE map reference — no
-    clone, no form, no moderation task (there is no text content). Idempotent
-    (only drafts match) and one-way (regressing to scratch does not
-    un-submit; the gallery's draft_status filter hides scratch maps anyway).
-    Called inside the caller's transaction; does not commit.
+    clone, no form. Reaching a submitted-tier status (in_progress /
+    ready_to_share) flips drafts to submitted; REGRESSING below it flips a
+    live-referenced (map_is_clone=false) submission back to draft — the
+    author never filled a consent form, so withdrawing their map must
+    un-publish it everywhere (/api/submissions has no draft_status filter,
+    so one-way would strand the entry there). Clone-backed submissions stay
+    one-way: those were deliberate, consented submissions.
+
+    Idempotent; called inside the caller's transaction; does not commit.
+    Returns the ids of rows flipped to submitted — callers MUST enqueue
+    moderate_submission_by_id for each AFTER commit: the gallery card
+    renders the map's name/description, so an unscored auto entry would let
+    an abusive map title sail past the nsfw filter.
     """
-    if public_id is None or draft_status not in [
-        s.value for s in SUBMITTED_DRAFT_STATUSES
-    ]:
-        return 0
-    drafts = session.exec(
+    if public_id is None:
+        return []
+    submitted_tier = draft_status in [s.value for s in SUBMITTED_DRAFT_STATUSES]
+    rows = session.exec(
         select(Submission)
         .join(FormConfig, col(FormConfig.portal_id) == Submission.portal_id)
         .where(
             and_(
                 col(Submission.map_public_id) == public_id,
-                col(Submission.status) == SubmissionStatus.draft,
                 col(FormConfig.collection_mode).in_(CollectionMode.auto_modes),
             )
         )
+        .with_for_update(of=Submission)
     ).all()
-    for draft in drafts:
-        draft.status = SubmissionStatus.submitted
-        draft.submitted_at = datetime.now(timezone.utc)
-        session.add(draft)
-    return len(drafts)
+    flipped: list[int] = []
+    for row in rows:
+        if submitted_tier and row.status == SubmissionStatus.draft:
+            row.status = SubmissionStatus.submitted
+            row.submitted_at = datetime.now(timezone.utc)
+            session.add(row)
+            flipped.append(row.id)
+        elif (
+            not submitted_tier
+            and row.status == SubmissionStatus.submitted
+            and not row.map_is_clone
+        ):
+            row.status = SubmissionStatus.draft
+            row.submitted_at = None
+            session.add(row)
+    return flipped
 
 
 def get_custom_fields(portal_id: str, session: Session) -> list[FormFieldCustom]:
@@ -541,15 +559,21 @@ async def flag_submission(
     session: Session = Depends(get_session),
 ):
     """Report a submission for reviewer attention. Only publicly visible
-    submissions can be flagged — flagging hidden ones gives moderators no
-    signal and is a way to harass the queue."""
+    submissions can be flagged — flagging hidden or internal-portal ones
+    gives moderators no signal, is a way to harass the queue, and would
+    leak an existence oracle for staff-only galleries."""
     submission = session.get(Submission, body.id)
     if (
         submission is None
         or submission.hidden
         or submission.status != SubmissionStatus.submitted
     ):
         raise HTTPException(status_code=404, detail="Submission not found")
+    config = session.exec(
+        select(FormConfig).where(col(FormConfig.portal_id) == submission.portal_id)
+    ).first()
+    if config is not None and config.collection_mode == CollectionMode.internal:
+        raise HTTPException(status_code=404, detail="Submission not found")
     submission.flagged = True
     session.add(submission)
     session.commit()

backend/app/submissions/models.py

@@ -120,7 +120,10 @@ class FormFieldCustom(TimeStampMixin, SQLModel, table=True):
 
     Keys are 'custom_'-prefixed (slugified from the label by the CMS) so they
     can never collide with registry field names; values are stored in
-    submissions_content like any other field and are PUBLIC.
+    submissions_content like any other field and are PUBLIC — there is no
+    per-question private bit, so the CMS question editor warns admins not to
+    collect contact info through custom questions (the registry's email
+    field is the private channel).
     """
 
     metadata = MetaData(schema=COMMENTS_SCHEMA)