lint-changelog re-confirms a release live before dropping its availability admonition

Author: kdeldycke

.github/workflows/autofix.yaml

@@ -641,8 +641,11 @@ jobs:
         with:
           version: "0.12.1"
       # Persist repomatic's HTTP cache (PyPI/GitHub/npm release metadata) across
-      # runs. Entries are TTL-gated in repomatic.cache, so a restored cache never
-      # serves a stale "latest" version; this mainly speeds up bursts of pushes.
+      # runs; this mainly speeds up bursts of pushes. TTL-gating bounds staleness
+      # at a day rather than preventing it, so a restored cache can miss a
+      # release published since it was written. Here that only defers a version
+      # bump by a run. A job that writes availability claims must re-confirm
+      # live instead: see the retraction gate in repomatic/changelog.py.
       - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/repomatic

.github/workflows/self-maintenance.yaml

@@ -44,8 +44,10 @@ jobs:
       - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
           version: "0.12.1"
-      # Persist repomatic's HTTP cache (PyPI/GitHub/npm release metadata) across runs. Entries are TTL-gated in
-      # repomatic.cache, so a restored cache never serves a stale "latest" version.
+      # Persist repomatic's HTTP cache (PyPI/GitHub/npm release metadata) across runs. TTL-gating bounds staleness at a
+      # day rather than preventing it, so a restored cache can miss a release published since it was written. Here that
+      # only defers a version bump by a run; a job that writes availability claims must re-confirm live, as the
+      # retraction gate in repomatic/changelog.py does.
       - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/repomatic

changelog.md

@@ -6,6 +6,7 @@
 > This version is **not released yet** and is under active development.
 
 - The release lane's `publish-release` job now checks out the repository, so uploading binaries to the release no longer fails with `Failed to spawn: repomatic`. `7.7.0` shipped with no standalone executables because of it.
+- `lint-changelog` re-confirms a release live before dropping its availability admonition, so a day-old cache no longer reports a just-published version as missing.
 - The readme's logo is now an absolute URL, so it renders on the PyPI project page instead of 404ing.
 - The install guide's executable table points at the last release that carries binaries, instead of a version whose upload lane failed.
 

repomatic/changelog.py

@@ -733,6 +733,64 @@ def replace_section(self, version: str, new_section: str) -> bool:
         return True
 
 
+def _load_pypi_releases(
+    package: str | None, history: Sequence[str], *, force_refresh: bool = False
+) -> dict[str, PyPIRelease]:
+    """Load a package's PyPI releases, merged with its former names.
+
+    A renamed project keeps its old releases under the old name, so the
+    changelog's early versions resolve only through
+    `[tool.repomatic] pypi.package-history`. The current package wins on
+    version collisions.
+
+    :param package: Current PyPI package name; empty or `None` skips PyPI.
+    :param history: Former package names, oldest-first.
+    :param force_refresh: Ignore cached responses and re-fetch.
+    :return: Dict mapping version strings to {class}`PyPIRelease` tuples.
+    """
+    releases: dict[str, PyPIRelease] = {}
+    if package:
+        releases = get_pypi_release_dates(package, force_refresh=force_refresh)
+    for former_package in history:
+        former_data = get_pypi_release_dates(
+            former_package, force_refresh=force_refresh
+        )
+        if former_data:
+            logging.info(
+                f"Using PyPI history for {former_package!r}"
+                f" ({len(former_data)} releases found)."
+            )
+        for version, release in former_data.items():
+            releases.setdefault(version, release)
+    return releases
+
+
+def _is_platform_gap(
+    parsed: Version,
+    on_platform: bool,
+    platform_known: bool,
+    first_version: Version | None,
+) -> bool:
+    """Whether *parsed* is missing from a platform it should appear on.
+
+    A gap is an absence *after* the platform's first release: versions
+    predating it were never expected there, so they are not warned about.
+
+    :param parsed: The version under test.
+    :param on_platform: Whether the lookup found it on the platform.
+    :param platform_known: Whether the platform is addressable at all (a
+        package name for PyPI, a repository URL for GitHub).
+    :param first_version: Earliest version seen on that platform.
+    :return: `True` when the version is a genuine gap.
+    """
+    return (
+        not on_platform
+        and platform_known
+        and first_version is not None
+        and parsed >= first_version
+    )
+
+
 def _platform_admonition(
     template: str, version: str, verb: str, platforms: Sequence[str]
 ) -> str:
@@ -985,33 +1043,20 @@ def lint_changelog_dates(
     if package is None:
         package = get_project_name()
 
-    # Fetch all PyPI release dates in a single API call.
-    pypi_data: dict[str, PyPIRelease] = {}
-    if package:
-        pypi_data = get_pypi_release_dates(package)
-        if pypi_data:
-            logging.info(
-                f"Using PyPI as reference for {package!r}"
-                f" ({len(pypi_data)} releases found)."
-            )
-        else:
-            logging.info(
-                f"Package {package!r} not found on PyPI, falling back to git tags."
-            )
-    else:
+    # Fetch all PyPI release dates in a single API call, merged with the
+    # releases of any former package name (for renamed projects).
+    pypi_data = _load_pypi_releases(package, pypi_package_history)
+    if not package:
         logging.info("No package name detected, falling back to git tags.")
-
-    # Merge releases from former package names (for renamed projects).
-    for former_package in pypi_package_history:
-        former_data = get_pypi_release_dates(former_package)
-        if former_data:
-            logging.info(
-                f"Using PyPI history for {former_package!r}"
-                f" ({len(former_data)} releases found)."
-            )
-        # Current package wins on version collisions.
-        for v, rel in former_data.items():
-            pypi_data.setdefault(v, rel)
+    elif pypi_data:
+        logging.info(
+            f"Using PyPI as reference for {package!r}"
+            f" ({len(pypi_data)} releases found)."
+        )
+    else:
+        logging.info(
+            f"Package {package!r} not found on PyPI, falling back to git tags."
+        )
 
     use_pypi = bool(pypi_data)
     has_mismatch = False
@@ -1051,6 +1096,98 @@ def lint_changelog_dates(
         first_github_version = min(Version(v) for v in github_releases)
         logging.info(f"First GitHub version: {first_github_version}")
 
+    def retracted_versions() -> set[str]:
+        """Versions whose section claims availability the lookups now deny.
+
+        Reads the enclosing scope on every call, so re-running it after a
+        forced refresh answers against the fresh data. An available platform
+        is rendered as a markdown link and a missing one as a bare label, so
+        the `[` prefix is what separates a claim of presence from one of
+        absence.
+        """
+        found = set()
+        for candidate, _candidate_date in releases:
+            existing = changelog.decompose_version(candidate).availability_admonition
+            parsed_candidate = Version(candidate)
+            drops_pypi = (
+                _is_platform_gap(
+                    parsed_candidate,
+                    candidate in pypi_data,
+                    bool(package),
+                    first_pypi_version,
+                )
+                and f"[{PYPI_LABEL}](" in existing
+            )
+            drops_github = (
+                _is_platform_gap(
+                    parsed_candidate,
+                    candidate in github_releases,
+                    bool(repo_url),
+                    first_github_version,
+                )
+                and f"[{GITHUB_LABEL}](" in existing
+            )
+            if drops_pypi or drops_github:
+                found.add(candidate)
+        return found
+
+    # Retraction gate: never demote an existing "is available" claim to
+    # "is **not available**" on cached data. Both lookups are TTL-cached for a
+    # day, so a snapshot taken before a release landed reports that release as
+    # missing, which is indistinguishable here from one that was never
+    # published. The sanity gates further down catch only a *wholly* empty
+    # result; a stale-but-populated snapshot walks past them, and the window
+    # where it lies is exactly the hours after a release, when this command is
+    # most likely to run. Adding availability stays cheap and cached; dropping
+    # it has to be confirmed live. Running before the date loop means the
+    # spurious "not found on PyPI" warning goes away with it.
+    retracted = retracted_versions()
+    if retracted:
+        logging.warning(
+            f"Cached lookups would retract availability for"
+            f" {', '.join(sorted(retracted))}; re-confirming live."
+        )
+        pypi_data = _load_pypi_releases(
+            package, pypi_package_history, force_refresh=True
+        )
+        first_pypi_version = min(Version(v) for v in pypi_data) if pypi_data else None
+        use_pypi = bool(pypi_data)
+        if repo_url:
+            try:
+                github_releases = get_github_releases(repo_url, force_refresh=True)
+            except GitHubReleasesUnavailable as exc:
+                logging.warning(f"Confirming GitHub lookup failed: {exc}")
+                if fix:
+                    msg = (
+                        f"Refusing to rewrite changelog: cached data would drop"
+                        f" the availability of {', '.join(sorted(retracted))},"
+                        f" and the confirming GitHub lookup failed ({exc})."
+                        f" Re-run when the GitHub API is reachable."
+                    )
+                    logging.error(msg)
+                    emit_annotation(AnnotationLevel.ERROR, msg)
+                    return 2
+            else:
+                first_github_version = (
+                    min(Version(v) for v in github_releases)
+                    if github_releases
+                    else None
+                )
+        still_missing = retracted_versions()
+        if still_missing:
+            # Confirmed against the live APIs: the release really is gone (a
+            # deleted GitHub release, a removed PyPI file). Retracting is then
+            # the correct repair, not a stale-cache artifact.
+            logging.warning(
+                f"Confirmed live: {', '.join(sorted(still_missing))} no longer"
+                " published where the changelog claims."
+            )
+        else:
+            logging.info(
+                "Live lookups confirm the existing availability claims;"
+                " the cache was stale."
+            )
+
     # Detect orphaned versions: present in external sources but missing
     # from the changelog.
     tag_versions = get_all_version_tags()
@@ -1321,17 +1458,11 @@ def lint_changelog_dates(
             # Build the WARNING admonition for platforms where missing.
             # Only warn about gaps: versions that postdate the first
             # release on that platform but are absent from it.
-            pypi_gap = (
-                not on_pypi
-                and bool(package)
-                and first_pypi_version is not None
-                and parsed >= first_pypi_version
+            pypi_gap = _is_platform_gap(
+                parsed, on_pypi, bool(package), first_pypi_version
             )
-            github_gap = (
-                not on_github
-                and bool(repo_url)
-                and first_github_version is not None
-                and parsed >= first_github_version
+            github_gap = _is_platform_gap(
+                parsed, on_github, bool(repo_url), first_github_version
             )
             warning = build_unavailable_admonition(
                 version,

repomatic/github/releases.py

@@ -206,6 +206,8 @@ def _cached_release_map(
     namespace: str,
     repo_url: str,
     key_for: Callable[[str], str | None],
+    *,
+    force_refresh: bool = False,
 ) -> dict[str, GitHubRelease]:
     """Fetch a repository's releases as a keyed map, through the HTTP cache.
 
@@ -218,6 +220,7 @@ def _cached_release_map(
     :param repo_url: Repository URL.
     :param key_for: Maps a raw tag name to the result key, or `None` to skip
         that release.
+    :param force_refresh: Ignore any cached map and re-fetch every page.
     :return: Dict mapping keys to {class}`GitHubRelease` tuples. Empty when
         the repository has no releases or *repo_url* does not parse to an
         `owner/repo` pair.
@@ -231,7 +234,7 @@ def _cached_release_map(
 
     cache_key = f"{owner}/{repo}"
     ttl = load_repomatic_config().cache.github_releases_ttl
-    cached = get_cached_response(namespace, cache_key, ttl)
+    cached = None if force_refresh else get_cached_response(namespace, cache_key, ttl)
     if cached is not None:
         try:
             data = json.loads(cached)
@@ -261,7 +264,9 @@ def _cached_release_map(
     return result
 
 
-def get_github_releases(repo_url: str) -> dict[str, GitHubRelease]:
+def get_github_releases(
+    repo_url: str, *, force_refresh: bool = False
+) -> dict[str, GitHubRelease]:
     """Get versions and dates for all GitHub releases.
 
     Fetches all releases via the GitHub API with pagination. Extracts
@@ -270,6 +275,9 @@ def get_github_releases(repo_url: str) -> dict[str, GitHubRelease]:
 
     :param repo_url: Repository URL (e.g.
         `https://github.com/user/repo`).
+    :param force_refresh: Ignore any cached map and re-fetch. A cached map
+        predating a release reports it as absent, so callers acting on an
+        absence around release time should re-confirm live.
     :return: Dict mapping version strings to {class}`GitHubRelease`
         tuples. Empty dict only when the repository genuinely has no
         releases (the API returned an empty page) or when `repo_url`
@@ -283,6 +291,7 @@ def get_github_releases(repo_url: str) -> dict[str, GitHubRelease]:
         "github-releases",
         repo_url,
         lambda tag: tag[1:] if tag.startswith("v") else None,
+        force_refresh=force_refresh,
     )
 
 

repomatic/http.py

@@ -116,6 +116,7 @@ def get_cached_json(
     *,
     ttl: int,
     log_label: str,
+    force_refresh: bool = False,
 ) -> Any | None:
     """GET *url* as JSON through the raw-response cache.
 
@@ -124,14 +125,23 @@ def get_cached_json(
     positive), and returned parsed. The caller keeps the caching policy: it
     picks the namespace, the cache key, and the TTL.
 
+    ```{note}
+    *force_refresh* skips the cache **read** but keeps the write, which is
+    what separates it from `ttl=0`: the latter also skips the store, so a
+    caller using it to bypass a stale entry would leave that entry in place
+    for the next reader. A forced refresh replaces it.
+    ```
+
     :param namespace: Cache namespace (like `"pypi"` or `"npm"`).
     :param key: Cache key within the namespace, usually the package name.
     :param url: The URL to fetch on a cache miss.
     :param ttl: Freshness TTL in seconds; `0` disables caching.
     :param log_label: Human-readable label for the debug log on failure.
+    :param force_refresh: Ignore any cached body and re-fetch, then store
+        the fresh response.
     :return: The parsed JSON value, or `None` on any fetch failure.
     """
-    cached = get_cached_response(namespace, key, ttl)
+    cached = None if force_refresh else get_cached_response(namespace, key, ttl)
     if cached is not None:
         try:
             return json.loads(cached)