fix(recovery): guarantee durable settlement

Author: amavashev

.github/workflows/ci.yml

@@ -14,3 +14,24 @@ jobs:
     uses: runcycles/.github/.github/workflows/ci-python.yml@v1
     with:
       mypy-target: runcycles
+
+  recovery-conformance:
+    name: Durable recovery conformance
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: runcycles/cycles-protocol
+          ref: 8a22803316d91b85e444f4fb40ff2b8426f11773
+          path: .cycles-protocol
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+      - run: python -m pip install -e ".[dev]"
+      - name: Run shared durable recovery scenarios
+        run: >-
+          python .cycles-protocol/scripts/run_client_recovery_conformance.py
+          --claim durable
+          --adapter python scripts/recovery_conformance_adapter.py

.github/workflows/python-publish.yml

@@ -20,6 +20,27 @@ permissions:
   contents: read
 
 jobs:
+  recovery-conformance:
+    name: Durable recovery conformance
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: runcycles/cycles-protocol
+          ref: 8a22803316d91b85e444f4fb40ff2b8426f11773
+          path: .cycles-protocol
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+      - run: python -m pip install -e ".[dev]"
+      - name: Run shared durable recovery scenarios
+        run: >-
+          python .cycles-protocol/scripts/run_client_recovery_conformance.py
+          --claim durable
+          --adapter python scripts/recovery_conformance_adapter.py
+
   build:
     name: Build distributions
     runs-on: ubuntu-latest
@@ -64,7 +85,7 @@ jobs:
 
   publish-to-testpypi:
     name: Publish to TestPyPI
-    needs: build
+    needs: [build, recovery-conformance]
     runs-on: ubuntu-latest
     if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
     environment:
@@ -88,7 +109,7 @@ jobs:
 
   publish-to-pypi:
     name: Publish to PyPI
-    needs: build
+    needs: [build, recovery-conformance]
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
     environment:
@@ -148,4 +169,4 @@ jobs:
           name: ${{ github.ref_name }}
           body: ${{ steps.notes.outputs.notes }}
           draft: false
-          prerelease: ${{ contains(github.ref_name, '-') }}
+          prerelease: ${{ contains(github.ref_name, '-') }}

CHANGELOG.md

@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.2] - 2026-07-29
+
+### Added
+
+- Bind all shared durable-recovery and guarantee-boundary scenarios from the
+  protocol repository into pull-request and release CI.
+- Expose `cycles_evidence` on `CommitResponse`.
+
+### Fixed
+
+- Persist known actual usage before the first commit request, recover expired
+  commits through `/v1/events`, and accept only exact HTTP 200/201
+  schema-valid commit/event responses as terminal success.
+- Use `v2-<sha256(exact UTF-8 reservation id)>.json` journal filenames, safely
+  migrate matching legacy records, and preserve collision-free cross-SDK
+  replay.
+- Retain durable settlement records for contradictory retryable 4xx envelopes,
+  and report heartbeat transport failures with their same-key retry or stop
+  disposition.
+
 ## [0.5.1] - 2026-07-27
 
 ### Fixed

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "runcycles"
-version = "0.5.1"
+version = "0.5.2"
 description = "Python AI agent budget control — enforce LLM cost limits, tool permissions, and multi-tenant policies before agent actions execute."
 readme = "README.md"
 license = "Apache-2.0"

runcycles/journal.py

@@ -89,9 +89,13 @@ def _restrict_permissions(path: Path, mode: int) -> None:
 
 
 def _safe_filename(reservation_id: str) -> str:
-    # ASCII-only, matching the TS/Java SDKs exactly: same-tenant clients in
-    # other languages settle records from this directory, and their discard()
-    # must compute the identical filename or the record replays forever.
+    """Cross-SDK, collision-resistant filename for an exact reservation id."""
+    digest = hashlib.sha256(reservation_id.encode("utf-8")).hexdigest()
+    return f"v2-{digest}{_SUFFIX}"
+
+
+def _legacy_filename(reservation_id: str) -> str:
+    """Filename written by SDK releases before the v2 digest scheme."""
     sanitized = re.sub(r"[^A-Za-z0-9_-]", "_", reservation_id)
     return f"{sanitized}{_SUFFIX}"
 
@@ -200,6 +204,15 @@ def discard(self, reservation_id: str) -> None:
         """Remove a journal entry after a terminal outcome. Never raises."""
         try:
             (self._dir / _safe_filename(reservation_id)).unlink(missing_ok=True)
+            legacy = self._dir / _legacy_filename(reservation_id)
+            if legacy.exists():
+                try:
+                    entry = PendingCommitRecord.from_json(legacy.read_text(encoding="utf-8"))
+                    if entry.reservation_id == reservation_id:
+                        legacy.unlink(missing_ok=True)
+                except (OSError, ValueError, KeyError, json.JSONDecodeError):
+                    # Never delete a colliding or malformed legacy record.
+                    pass
         except OSError:
             logger.warning("Failed to discard journal entry: id=%s", reservation_id, exc_info=True)
 
@@ -235,6 +248,38 @@ def load_pending(self, base_url: str) -> list[PendingCommitRecord]:
                     except OSError:
                         pass
                     continue
+                standard_path = self._dir / _safe_filename(entry.reservation_id)
+                duplicate_of_standard = False
+                if path != standard_path:
+                    try:
+                        if not standard_path.exists():
+                            path.replace(standard_path)
+                            logger.info(
+                                "Migrated legacy journal filename: id=%s, path=%s",
+                                entry.reservation_id,
+                                standard_path,
+                            )
+                        else:
+                            existing = PendingCommitRecord.from_json(
+                                standard_path.read_text(encoding="utf-8")
+                            )
+                            if existing.reservation_id == entry.reservation_id:
+                                path.unlink(missing_ok=True)
+                                duplicate_of_standard = True
+                                logger.info(
+                                    "Removed duplicate legacy journal filename: id=%s, path=%s",
+                                    entry.reservation_id,
+                                    path,
+                                )
+                    except (OSError, ValueError, KeyError, json.JSONDecodeError):
+                        logger.warning(
+                            "Could not safely migrate legacy journal filename: id=%s, path=%s",
+                            entry.reservation_id,
+                            path,
+                            exc_info=True,
+                        )
+                if duplicate_of_standard:
+                    continue
                 if entry.base_url == base_url:
                     entries.append(entry)
         except OSError:

runcycles/lifecycle.py

@@ -46,6 +46,7 @@
     CommitRetryEngine,
     _extract_error_code,
     _is_recognized_rejection,
+    _is_schema_valid_commit_success,
 )
 
 logger = logging.getLogger(__name__)
@@ -763,11 +764,23 @@ def _handle_commit(
         commit_body: dict[str, Any],
         event_fallback_body: dict[str, Any],
     ) -> None:
+        self._retry_engine.persist_pending(
+            reservation_id, commit_body, event_fallback_body
+        )
         try:
             logger.debug("Committing: id=%s", reservation_id)
             response = self._client.commit_reservation(reservation_id, commit_body)
-            if response.is_success:
+            if _is_schema_valid_commit_success(response):
+                self._retry_engine.discard_pending(reservation_id)
                 logger.info("Commit successful: id=%s", reservation_id)
+            elif response.is_success:
+                logger.warning(
+                    "Commit returned ambiguous protocol-invalid 2xx; scheduling same-key retry: "
+                    "id=%s, status=%d",
+                    reservation_id,
+                    response.status,
+                )
+                self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
             elif response.is_transport_error or response.is_server_error:
                 logger.warning("Commit failed (retryable): id=%s, status=%d", reservation_id, response.status)
                 self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
@@ -804,10 +817,13 @@ def _handle_commit(
                     )
                     self._retry_engine.schedule_event(reservation_id, event_fallback_body)
                 elif error_code == "RESERVATION_FINALIZED":
+                    self._retry_engine.discard_pending(reservation_id)
                     logger.warning("Reservation already finalized: id=%s", reservation_id)
                 elif error_code == "IDEMPOTENCY_MISMATCH":
+                    self._retry_engine.discard_pending(reservation_id)
                     logger.warning("Commit idempotency mismatch (not releasing): id=%s", reservation_id)
                 elif response.is_client_error and _is_recognized_rejection(error_code):
+                    self._retry_engine.discard_pending(reservation_id)
                     self._handle_release(reservation_id, f"commit_rejected_{error_code}")
                 elif response.is_client_error:
                     # Codeless or forward-compat-unknown 4xx: neither release
@@ -820,7 +836,12 @@ def _handle_commit(
                     )
                     self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
                 else:
-                    logger.warning("Unrecognized commit response: id=%s, response=%s", reservation_id, response)
+                    logger.warning(
+                        "Unrecognized commit response; scheduling same-key retry: id=%s, response=%s",
+                        reservation_id,
+                        response,
+                    )
+                    self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
         except Exception:
             logger.exception("Failed to commit: id=%s", reservation_id)
             self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
@@ -1047,16 +1068,24 @@ def heartbeat_loop() -> None:
                                 return
                             delay_ms = nxt
                 except Exception:
-                    logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True)
                     if authoritative:
                         nxt = sched.on_transient_failure(_now_mono_ms())
                         if nxt is None:
                             logger.warning(
-                                "Heartbeat stopping: no safe recovery window remains: id=%s",
+                                "Heartbeat extend transport error; stopping because no safe recovery "
+                                "window remains: id=%s",
                                 reservation_id,
+                                exc_info=True,
                             )
                             return
                         delay_ms = nxt
+                    logger.warning(
+                        "Heartbeat extend transport error; retrying with the same idempotency key "
+                        "in %.0fms: id=%s",
+                        delay_ms,
+                        reservation_id,
+                        exc_info=True,
+                    )
 
         t = threading.Thread(target=heartbeat_loop, daemon=True, name=f"cycles-heartbeat-{reservation_id[:12]}")
         t.start()
@@ -1189,10 +1218,22 @@ async def _handle_commit(
         commit_body: dict[str, Any],
         event_fallback_body: dict[str, Any],
     ) -> None:
+        self._retry_engine.persist_pending(
+            reservation_id, commit_body, event_fallback_body
+        )
         try:
             response = await self._client.commit_reservation(reservation_id, commit_body)
-            if response.is_success:
+            if _is_schema_valid_commit_success(response):
+                self._retry_engine.discard_pending(reservation_id)
                 logger.info("Commit successful: id=%s", reservation_id)
+            elif response.is_success:
+                logger.warning(
+                    "Commit returned ambiguous protocol-invalid 2xx; scheduling same-key retry: "
+                    "id=%s, status=%d",
+                    reservation_id,
+                    response.status,
+                )
+                self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
             elif response.is_transport_error or response.is_server_error:
                 self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
             else:
@@ -1228,10 +1269,13 @@ async def _handle_commit(
                     )
                     self._retry_engine.schedule_event(reservation_id, event_fallback_body)
                 elif error_code == "RESERVATION_FINALIZED":
+                    self._retry_engine.discard_pending(reservation_id)
                     logger.warning("Reservation already finalized: id=%s", reservation_id)
                 elif error_code == "IDEMPOTENCY_MISMATCH":
+                    self._retry_engine.discard_pending(reservation_id)
                     logger.warning("Commit idempotency mismatch (not releasing): id=%s", reservation_id)
                 elif response.is_client_error and _is_recognized_rejection(error_code):
+                    self._retry_engine.discard_pending(reservation_id)
                     await self._handle_release(reservation_id, f"commit_rejected_{error_code}")
                 elif response.is_client_error:
                     # Codeless or forward-compat-unknown 4xx: neither release
@@ -1244,7 +1288,12 @@ async def _handle_commit(
                     )
                     self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
                 else:
-                    logger.warning("Unrecognized commit response: id=%s, response=%s", reservation_id, response)
+                    logger.warning(
+                        "Unrecognized commit response; scheduling same-key retry: id=%s, response=%s",
+                        reservation_id,
+                        response,
+                    )
+                    self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
         except Exception:
             logger.exception("Failed to commit: id=%s", reservation_id)
             self._retry_engine.schedule(reservation_id, commit_body, event_fallback_body)
@@ -1457,16 +1506,24 @@ async def heartbeat_loop() -> None:
                                     return
                                 delay_ms = nxt
                     except Exception:
-                        logger.warning("Heartbeat extend error: id=%s", reservation_id, exc_info=True)
                         if authoritative:
                             nxt = sched.on_transient_failure(_now_mono_ms())
                             if nxt is None:
                                 logger.warning(
-                                    "Heartbeat stopping: no safe recovery window remains: id=%s",
+                                    "Heartbeat extend transport error; stopping because no safe "
+                                    "recovery window remains: id=%s",
                                     reservation_id,
+                                    exc_info=True,
                                 )
                                 return
                             delay_ms = nxt
+                        logger.warning(
+                            "Heartbeat extend transport error; retrying with the same idempotency "
+                            "key in %.0fms: id=%s",
+                            delay_ms,
+                            reservation_id,
+                            exc_info=True,
+                        )
             except asyncio.CancelledError:
                 return
 

runcycles/models.py

@@ -309,6 +309,7 @@ class CommitResponse(BaseModel):
     charged: Amount
     released: Amount | None = None
     balances: list[Balance] | None = None
+    cycles_evidence: CyclesEvidenceRef | None = None
 
 
 class ReleaseResponse(BaseModel):