Add revert risk model check to autoreview

app/reviews/admin.py

@@ -11,8 +11,9 @@ class WikiAdmin(admin.ModelAdmin):
 
 @admin.register(WikiConfiguration)
 class WikiConfigurationAdmin(admin.ModelAdmin):
-    list_display = ("wiki", "updated_at")
+    list_display = ("wiki", "revertrisk_threshold", "updated_at")
     search_fields = ("wiki__name", "wiki__code")
+    fields = ["wiki", "revertrisk_threshold"]
 
 
 @admin.register(PendingPage)

app/reviews/autoreview.py

@@ -2,6 +2,14 @@
 
 from __future__ import annotations
 
+
+import json
+import logging
+from dataclasses import dataclass
+from typing import Iterable
+
+from .models import EditorProfile, PendingPage, PendingRevision
+from pywikibot.comms import http
 import logging
 import re
 from collections.abc import Iterable
@@ -41,6 +49,7 @@ def run_autoreview_for_page(page: PendingPage) -> list[dict]:
 
     auto_groups = _normalize_to_lookup(configuration.auto_approved_groups)
     blocking_categories = _normalize_to_lookup(configuration.blocking_categories)
+    revertrisk_threshold = getattr(configuration, "revertrisk_threshold", None)
     redirect_aliases = _get_redirect_aliases(page.wiki)
     client = WikiClient(page.wiki)
 
@@ -53,6 +62,7 @@ def run_autoreview_for_page(page: PendingPage) -> list[dict]:
             profile,
             auto_groups=auto_groups,
             blocking_categories=blocking_categories,
+            revertrisk_threshold=revertrisk_threshold,
             redirect_aliases=redirect_aliases,
         )
         results.append(
@@ -77,6 +87,7 @@ def _evaluate_revision(
     *,
     auto_groups: dict[str, str],
     blocking_categories: dict[str, str],
+    revertrisk_threshold: float | None,
     redirect_aliases: list[str],
 ) -> dict:
     tests: list[dict] = []
@@ -377,6 +388,46 @@ def _evaluate_revision(
         }
     )
 
+    # Test 4: High revert risk prevents automatic approval.
+    if revertrisk_threshold is not None:
+        score = _get_revertrisk_score(revision)
+
+        if score is None:
+            tests.append(
+                {
+                    "id": "revertrisk",
+                    "title": "Revert risk check",
+                    "status": "error",
+                    "message": "Could not retrieve revert risk score.",
+                }
+            )
+        elif score > revertrisk_threshold:
+            tests.append(
+                {
+                    "id": "revertrisk",
+                    "title": "Revert risk check",
+                    "status": "fail",
+                    "message": f"High revert risk: {score:.3f} exceeds threshold {revertrisk_threshold:.3f}.",
+                }
+            )
+            return {
+                "tests": tests,
+                "decision": AutoreviewDecision(
+                    status="blocked",
+                    label="Cannot be auto-approved",
+                    reason="The edit has a high probability of being reverted.",
+                ),
+            }
+        else:
+            tests.append(
+                {
+                    "id": "revertrisk",
+                    "title": "Revert risk check",
+                    "status": "ok",
+                    "message": f"Revert risk {score:.3f} is below threshold {revertrisk_threshold:.3f}.",
+                }
+            )
+    # Test 4: Check for new rendering errors in the HTML.
     # Test 8: Check for new rendering errors in the HTML.
     new_render_errors = _check_for_new_render_errors(revision, client)
     if new_render_errors:
@@ -447,6 +498,35 @@ def _evaluate_revision(
         ),
     }
 
+def _get_revertrisk_score(revision: PendingRevision) -> float | None:
+    """Query the Wikimedia revertrisk API to get the revert risk score for an edit."""
+    try:
+        lang = getattr(revision.page.wiki, "language_code", "en")
+
+        url = "https://api.wikimedia.org/service/lw/inference/v1/models/revertrisk-multilingual:predict"
+        headers = {
+            "Content-Type": "application/json",
+            "User-Agent": "PendingChangesBot/1.0 (https://fi.wikipedia.org/wiki/User:SeulojaBot)",
+        }
+        payload = json.dumps({"rev_id": revision.revid, "lang": lang})
+
+        resp = http.fetch(url, method="POST", headers=headers, data=payload)
+
+        if resp.status_code == 200:
+            data = json.loads(resp.text)
+            # handle API response consistently
+            if "score" in data:
+                return float(data["score"])
+            # fallback for nested probabilities
+            return float(data.get("output", {}).get("probabilities", {}).get("true", 0.0))
+
+        logger.warning(f"Revertrisk API returned {resp.status_code} for {revision.revid}")
+        return None
+
+    except Exception as e:
+        logger.exception(f"Error fetching revertrisk for {revision.revid}: {e}")
+        return None
+
 
 def _get_render_error_count(revision: PendingRevision, html: str) -> int:
     """Calculate and cache the number of rendering errors in the HTML."""

app/reviews/models.py

@@ -60,6 +60,12 @@ class WikiConfiguration(models.Model):
     )
     updated_at = models.DateTimeField(auto_now=True)
 
+    revertrisk_threshold = models.FloatField(
+        null=True,
+        blank=True,
+        help_text="Revert risk threshold (0.0-1.0). Set to None to disable check."
+    )
+
     def __str__(self) -> str:  # pragma: no cover - debug helper
         return f"Configuration for {self.wiki.code}"
 

app/reviews/tests/test_autoreview.py

@@ -1,18 +1,99 @@
 from __future__ import annotations
 
+import unittest
+import logging
 from datetime import datetime
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch, MagicMock
 
 from django.test import TestCase
 
 from reviews import autoreview
 from reviews.autoreview import (
+    _evaluate_revision,
+    _get_revertrisk_score,
     _find_invalid_isbns,
     _validate_isbn_10,
     _validate_isbn_13,
 )
 from reviews.services import was_user_blocked_after
 
+logging.disable(logging.CRITICAL)
+
+
+class DummyRevision:
+    def __init__(self, revid=12345, user_name="TestUser", superset_data=None):
+        self.revid = revid
+        self.user_name = user_name
+        self.page = MagicMock()
+        self.page.categories = []
+        self.page.wiki = MagicMock()
+        self.page.wiki.language_code = "en"
+        self.superset_data = superset_data or {}
+
+    def get_categories(self):
+        return []
+
+class TestRevertrisk(unittest.TestCase):
+
+    @patch("reviews.autoreview.is_bot_edit", return_value=False)
+    @patch("reviews.autoreview._get_revertrisk_score")
+    def test_high_risk_blocks(self, mock_score, mock_bot_edit):
+        mock_score.return_value = 0.9
+        revision = DummyRevision()
+        result = _evaluate_revision(
+            revision, None, auto_groups={}, 
+            blocking_categories={}, revertrisk_threshold=0.8
+        )
+        self.assertEqual(result["decision"].status, "blocked")
+
+    @patch("reviews.autoreview.is_bot_edit", return_value=False)
+    @patch("reviews.autoreview._get_revertrisk_score")
+    def test_low_risk_continues(self, mock_score, mock_bot_edit):
+        mock_score.return_value = 0.5
+        revision = DummyRevision()
+        result = _evaluate_revision(
+            revision, None, auto_groups={}, 
+            blocking_categories={}, revertrisk_threshold=0.8
+        )
+        self.assertEqual(result["decision"].status, "manual")
+
+    @patch("reviews.autoreview.is_bot_edit", return_value=False)
+    @patch("reviews.autoreview._get_revertrisk_score")
+    def test_api_error_handled(self, mock_score, mock_bot_edit):
+        mock_score.return_value = None
+        revision = DummyRevision()
+        result = _evaluate_revision(
+            revision, None, auto_groups={}, 
+            blocking_categories={}, revertrisk_threshold=0.8
+        )
+        self.assertEqual(result["decision"].status, "manual")
+
+    @patch("reviews.autoreview._get_revertrisk_score")
+    def test_bot_bypasses_check(self, mock_score):
+        revision = DummyRevision(superset_data={"rc_bot": True})
+        result = _evaluate_revision(
+            revision, None, auto_groups={}, 
+            blocking_categories={}, revertrisk_threshold=0.8
+        )
+        self.assertEqual(result["decision"].status, "approve")
+        mock_score.assert_not_called()
+
+    @patch("reviews.autoreview.is_bot_edit", return_value=False)
+    def test_no_threshold_skips_check(self, mock_bot_edit):
+        revision = DummyRevision()
+        result = _evaluate_revision(
+            revision, None, auto_groups={}, 
+            blocking_categories={}, revertrisk_threshold=None
+        )
+        self.assertFalse(any(t["id"] == "revertrisk" for t in result["tests"]))
+
+    @patch("reviews.autoreview.http.fetch")
+    @patch("reviews.autoreview.is_bot_edit", return_value=False)
+    def test_api_exception_returns_none(self, mock_bot_edit, mock_fetch):
+        mock_fetch.side_effect = Exception("Network error")
+        score = _get_revertrisk_score(DummyRevision())
+        self.assertIsNone(score)
+
 
 class ISBNValidationTests(TestCase):
     """Test ISBN-10 and ISBN-13 checksum validation."""

app/reviews/views.py

@@ -367,28 +367,56 @@ def api_clear_cache(request: HttpRequest, pk: int) -> JsonResponse:
 @csrf_exempt
 @require_http_methods(["GET", "PUT"])
 def api_configuration(request: HttpRequest, pk: int) -> JsonResponse:
+    """Get or update wiki configuration including revertrisk threshold."""
     wiki = _get_wiki(pk)
     configuration = wiki.configuration
+
     if request.method == "PUT":
         if request.content_type == "application/json":
             payload = json.loads(request.body.decode("utf-8")) if request.body else {}
         else:
             payload = request.POST.dict()
+
         blocking_categories = payload.get("blocking_categories", [])
         auto_groups = payload.get("auto_approved_groups", [])
+
         if isinstance(blocking_categories, str):
             blocking_categories = [blocking_categories]
         if isinstance(auto_groups, str):
             auto_groups = [auto_groups]
+
+        # Handle revertrisk_threshold: empty string or null becomes None
+        revertrisk_threshold = payload.get("revertrisk_threshold")
+        if revertrisk_threshold is not None:
+            if revertrisk_threshold == "" or revertrisk_threshold == "null":
+                revertrisk_threshold = None
+            else:
+                try:
+                    revertrisk_threshold = float(revertrisk_threshold)
+                    if not (0.0 <= revertrisk_threshold <= 1.0):
+                        return JsonResponse(
+                            {"error": "revertrisk_threshold must be between 0.0 and 1.0"},
+                            status=400
+                        )
+                except (ValueError, TypeError):
+                    return JsonResponse(
+                        {"error": "revertrisk_threshold must be a valid number or null"},
+                        status=400
+                    )
+
         configuration.blocking_categories = blocking_categories
         configuration.auto_approved_groups = auto_groups
+        configuration.revertrisk_threshold = revertrisk_threshold
         configuration.save(
-            update_fields=["blocking_categories", "auto_approved_groups", "updated_at"]
+            update_fields=["blocking_categories", "auto_approved_groups", 
+                          "revertrisk_threshold", "updated_at"]
         )
+
     return JsonResponse(
         {
             "blocking_categories": configuration.blocking_categories,
             "auto_approved_groups": configuration.auto_approved_groups,
+            "revertrisk_threshold": configuration.revertrisk_threshold,
         }
     )
 

app/static/reviews/app.js

@@ -160,6 +160,7 @@ createApp({
     const forms = reactive({
       blockingCategories: "",
       autoApprovedGroups: "",
+      revertriskThreshold: "",
     });
 
     const currentWiki = computed(() =>
@@ -237,10 +238,14 @@ createApp({
       if (!currentWiki.value) {
         forms.blockingCategories = "";
         forms.autoApprovedGroups = "";
+        forms.revertriskThreshold = "";
         return;
       }
       forms.blockingCategories = (currentWiki.value.configuration.blocking_categories || []).join("\n");
       forms.autoApprovedGroups = (currentWiki.value.configuration.auto_approved_groups || []).join("\n");
+      forms.revertriskThreshold = currentWiki.value.configuration.revertrisk_threshold !== null
+        ? String(currentWiki.value.configuration.revertrisk_threshold)
+        : "";
     }
 
     async function apiRequest(url, options = {}) {
@@ -355,9 +360,20 @@ createApp({
       if (!state.selectedWikiId) {
         return;
       }
+
+      let revertriskValue = null;
+      if (forms.revertriskThreshold && forms.revertriskThreshold.trim() !== "") {
+        revertriskValue = parseFloat(forms.revertriskThreshold);
+        if (isNaN(revertriskValue)) {
+          state.error = "Revertrisk threshold must be a valid number";
+          return;
+        }
+      }
+
       const payload = {
         blocking_categories: parseTextarea(forms.blockingCategories),
         auto_approved_groups: parseTextarea(forms.autoApprovedGroups),
+        revertrisk_threshold: revertriskValue,
       };
       try {
         const data = await apiRequest(`/api/wikis/${state.selectedWikiId}/configuration/`, {