fix: fail fast on provider-wide X outage (#61)

Author: hiidkaboutyou-spec

app/main.py

@@ -491,6 +491,13 @@ async def run_scheduled_scan(self) -> None:
         self.state.data["last_auto_attempt"] = now.isoformat()
         lookback = max(2, int(self.settings.runtime.get("scheduled_lookback_hours", 24)))
         start = max(last - timedelta(minutes=30), now - timedelta(hours=lookback))
+        if getattr(self.collector, "provider_preflight_blocked", lambda: False)():
+            logger.warning(
+                "Scheduled X scan skipped because the immediately preceding provider probe was offline."
+            )
+            self.state.data["last_failed_sources"] = list(self.collector.last_errors)[:10]
+            self._record_x_scan_failure(now)
+            return
         try:
             updates = await self.collector.collect_window(start, now, max_per_query=200)
         except XCollectionError as exc:

app/production_preflight.py

@@ -2,6 +2,8 @@
 
 import asyncio
 import logging
+import os
+from pathlib import Path
 
 from .config import ConfigError, Settings
 from .telegram import TelegramBot, TelegramError
@@ -10,6 +12,17 @@
 logger = logging.getLogger(__name__)
 
 
+def _publish_github_provider_state(report: dict[str, str]) -> None:
+    """Expose the live X probe to later steps without making X a hard dependency."""
+    github_env = os.environ.get("GITHUB_ENV", "").strip()
+    if not github_env:
+        return
+    x_status = str(report.get("x", "offline (missing status)"))
+    state = "online" if x_status == "ok" else "offline"
+    with Path(github_env).open("a", encoding="utf-8") as stream:
+        stream.write(f"X_PROVIDER_PREFLIGHT={state}\n")
+
+
 def _check_telegram(settings: Settings) -> str:
     """Fail closed only when the assistant cannot receive or send messages."""
     telegram = TelegramBot(
@@ -92,6 +105,7 @@ def main() -> int:
 
     for provider, status in report.items():
         print(f"Production preflight: {provider}={status}")
+    _publish_github_provider_state(report)
     return 0
 
 

app/webhook_aware_assistant.py

@@ -168,6 +168,13 @@ async def run_scheduled_scan(self) -> None:
 
         lookback = max(2, int(self.settings.runtime.get("scheduled_lookback_hours", 24)))
         start = max(last - timedelta(minutes=30), now - timedelta(hours=lookback))
+        if getattr(self.collector, "provider_preflight_blocked", lambda: False)():
+            logger.warning(
+                "Scheduled X scan skipped because the immediately preceding provider probe was offline."
+            )
+            self.state.data["last_failed_sources"] = list(self.collector.last_errors)[:10]
+            self._record_x_scan_failure(now)
+            return
         try:
             updates = await self.collector.collect_window(start, now, max_per_query=200)
         except XCollectionError as exc:

app/x_client.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import logging
+import os
 import re
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -138,6 +139,20 @@ def __init__(self, cookies: dict[str, str], sources: list[dict[str, Any]], keywo
             and str(item.get("mode", "")).strip().lower() == SourceMode.FULL_FEED.value
         }
 
+    def provider_preflight_blocked(self) -> bool:
+        """Stop a known provider-wide outage before retrying every source."""
+        if os.environ.get("X_PROVIDER_PREFLIGHT", "").strip().lower() != "offline":
+            return False
+        errors: list[str] = []
+        for source in self.sources:
+            if not source.get("enabled", True):
+                continue
+            handle = normalize_handle(str(source.get("handle", "")))
+            if handle:
+                errors.append(f"@{handle}: provider_preflight_offline")
+        self.last_errors = errors or ["provider_preflight_offline"]
+        return True
+
     async def _get_api(self):
         if self.api is not None:
             return self.api

docs/LAUNCH_STATUS.md

@@ -1,6 +1,19 @@
 # Production Launch Status
 
-آخرین به‌روزرسانی: 2026-08-12
+آخرین به‌روزرسانی: 2026-09-07
+
+## هشدار عملیاتی فعلی
+
+- اجرای اصلی Daily روی GitHub Actions از 2026-09-06/07 در لبهٔ `x.com` با پاسخ
+  Cloudflare `403` روبه‌رو است؛ این failure پیش از دریافت GraphQL و مستقل از اعتبار cookie
+  مشاهده شده است.
+- Nightly Fanfic Digest در همین بازه سالم اجرا شده، بنابراین outage به runtime اصلی X
+  محدود است و به‌عنوان COMPLETE یا پنجرهٔ خالی معتبر ثبت نمی‌شود.
+- نسخهٔ curl/browser-TLS کتابخانه نیز روی GitHub-hosted runner همان Cloudflare challenge را
+  دریافت کرده است؛ بنابراین ارتقای dependency به‌تنهایی recovery محسوب نمی‌شود.
+- تا زمان اثبات یک egress سالم، preflight آفلاین باید اسکن X را fail-fast کند، cursor را جلو
+  نبرد و همهٔ منابع را صریحاً failed/unproven نگه دارد. آزمایش میزبان جایگزین فقط به‌صورت
+  ناشناس و بدون انتقال cookie/secret انجام می‌شود.
 
 ## وضعیت
 

tests/test_audit_regressions.py

@@ -135,6 +135,23 @@ def test_explicit_live_part_numbers_override_bad_timestamps(self):
 
 
 class XAuditTests(unittest.TestCase):
+    def test_offline_provider_preflight_names_every_enabled_source(self):
+        collector = XCollector(
+            {},
+            [
+                {"handle": "alpha", "enabled": True},
+                {"handle": "beta", "enabled": True},
+                {"handle": "disabled", "enabled": False},
+            ],
+            [],
+        )
+        with patch.dict(os.environ, {"X_PROVIDER_PREFLIGHT": "offline"}):
+            self.assertTrue(collector.provider_preflight_blocked())
+        self.assertEqual(
+            collector.last_errors,
+            ["@alpha: provider_preflight_offline", "@beta: provider_preflight_offline"],
+        )
+
     def test_keyword_synonyms_are_combined_into_one_query_per_group(self):
         queries = _keyword_queries(
             [

tests/test_production_preflight.py

@@ -1,12 +1,20 @@
 from __future__ import annotations
 
 import asyncio
+import os
+import tempfile
 import unittest
+from pathlib import Path
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, patch
 
 from app.config import ConfigError
-from app.production_preflight import _check_gemini, _check_telegram, _check_x
+from app.production_preflight import (
+    _check_gemini,
+    _check_telegram,
+    _check_x,
+    _publish_github_provider_state,
+)
 from app.telegram import TelegramPermanentError
 from app.x_client import XCollectionError
 
@@ -27,6 +35,20 @@ def settings(**overrides):
 
 
 class ProductionPreflightTests(unittest.TestCase):
+    def test_github_env_records_offline_x_without_error_detail(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "github-env"
+            with patch.dict(os.environ, {"GITHUB_ENV": str(path)}):
+                _publish_github_provider_state({"x": "offline (XCollectionError)"})
+            self.assertEqual(path.read_text(encoding="utf-8"), "X_PROVIDER_PREFLIGHT=offline\n")
+
+    def test_github_env_records_online_x(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "github-env"
+            with patch.dict(os.environ, {"GITHUB_ENV": str(path)}):
+                _publish_github_provider_state({"x": "ok"})
+            self.assertEqual(path.read_text(encoding="utf-8"), "X_PROVIDER_PREFLIGHT=online\n")
+
     @patch("app.production_preflight.TelegramBot")
     def test_telegram_is_the_hard_dependency(self, bot_class):
         bot_class.return_value.api.side_effect = TelegramPermanentError("bad token")

tests/test_webhook_runtime.py

@@ -78,6 +78,36 @@ def test_partial_x_attempt_is_persistently_throttled_between_action_passes(self)
 
         app.collector.collect_window.assert_not_awaited()
 
+    def test_provider_wide_preflight_outage_skips_collection_without_advancing_cursor(self):
+        now = datetime.now(timezone.utc)
+        previous_cursor = (now - timedelta(hours=1)).isoformat()
+        app = object.__new__(WebhookAwarePersonalAssistant)
+        app.state = SimpleNamespace(
+            data={
+                "last_auto_run": previous_cursor,
+                "last_auto_attempt": (now - timedelta(minutes=30)).isoformat(),
+            }
+        )
+        app.settings = SimpleNamespace(
+            runtime={"scheduled_min_interval_minutes": 12, "scheduled_lookback_hours": 24}
+        )
+        app.collector = SimpleNamespace(
+            provider_preflight_blocked=Mock(return_value=True),
+            collect_window=AsyncMock(),
+            last_errors=["@alpha: provider_preflight_offline"],
+        )
+        app._notify_x_failure_if_due = Mock()
+
+        asyncio.run(app.run_scheduled_scan())
+
+        app.collector.collect_window.assert_not_awaited()
+        self.assertEqual(app.state.data["last_auto_run"], previous_cursor)
+        self.assertEqual(app.state.data["x_scan_failure_streak"], 1)
+        self.assertEqual(
+            app.state.data["last_failed_sources"],
+            ["@alpha: provider_preflight_offline"],
+        )
+
     def test_partial_source_fetch_queues_recovery_but_does_not_advance_success_cursor(self):
         now = datetime.now(timezone.utc)
         previous_cursor = (now - timedelta(hours=1)).isoformat()