Exclude deprecated apps from catalog and add URL health check

.github/scripts/check_download_urls.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Check that every catalog download URL still resolves to a downloadable file.
+
+Vendors sometimes delete versioned files and redirect to an HTML error page
+that is served with a 200 status. The nightly collector only syncs version
+data from Homebrew, so a rotted URL stays in the catalog silently until a
+user upload fails. This script probes every non-deprecated app URL and
+writes a markdown report of the broken ones, so a scheduled workflow can
+open or update a GitHub issue.
+
+Exit code is always 0. The number of broken URLs is exposed through the
+GITHUB_OUTPUT variable "broken_count" and the report is written to
+url-health-report.md in the working directory.
+"""
+
+import concurrent.futures
+import datetime
+import json
+import os
+import sys
+
+import requests
+
+APPS_FOLDER = "Apps"
+REPORT_FILE = "url-health-report.md"
+TIMEOUT_SECONDS = 45
+MAX_WORKERS = 20
+# Different CDNs block different clients: SourceForge rejects browser user
+# agents from non-browser TLS stacks, Tableau's CDN only allows curl-style
+# agents, others require a browser agent. Try them in order and treat the
+# URL as broken only if every attempt fails.
+ATTEMPTS = [
+    {"method": "HEAD", "user_agent": "IntuneBrew-URL-Health-Check/1.0"},
+    {"method": "GET", "user_agent": "IntuneBrew-URL-Health-Check/1.0"},
+    {"method": "GET", "user_agent": "curl/8.4.0"},
+    {
+        "method": "GET",
+        "user_agent": (
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+            "(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
+        ),
+    },
+]
+
+
+def check_url(url):
+    """Return (verdict, detail). Verdict is 'ok' or a failure category."""
+    verdict, detail = "request_failed", "no attempt succeeded"
+    for attempt in ATTEMPTS:
+        headers = {"User-Agent": attempt["user_agent"]}
+        if attempt["method"] == "GET":
+            headers["Range"] = "bytes=0-0"
+        try:
+            response = requests.request(
+                attempt["method"],
+                url,
+                headers=headers,
+                timeout=TIMEOUT_SECONDS,
+                allow_redirects=True,
+                stream=True,
+            )
+            response.close()
+        except requests.RequestException as e:
+            verdict, detail = "request_failed", type(e).__name__
+            continue
+
+        if response.status_code >= 400:
+            verdict, detail = "http_error", f"HTTP {response.status_code}"
+            continue
+
+        content_type = response.headers.get("content-type", "")
+        if "text/html" in content_type:
+            verdict, detail = "html_page", f"returns an HTML page (final URL: {response.url})"
+            continue
+
+        return "ok", content_type
+
+    return verdict, detail
+
+
+def main():
+    apps = []
+    for filename in sorted(os.listdir(APPS_FOLDER)):
+        if not filename.endswith(".json"):
+            continue
+        with open(os.path.join(APPS_FOLDER, filename)) as f:
+            try:
+                data = json.load(f)
+            except ValueError:
+                print(f"Skipping unparseable file: {filename}")
+                continue
+        if data.get("deprecated"):
+            continue
+        if not data.get("url"):
+            continue
+        apps.append({"name": filename[: -len(".json")], "version": data.get("version", ""), "url": data["url"]})
+
+    print(f"Checking {len(apps)} download URLs...")
+    broken = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        futures = {pool.submit(check_url, app["url"]): app for app in apps}
+        for future in concurrent.futures.as_completed(futures):
+            app = futures[future]
+            verdict, detail = future.result()
+            if verdict != "ok":
+                broken.append({**app, "verdict": verdict, "detail": detail})
+                print(f"BROKEN {app['name']}: {verdict} ({detail})")
+
+    broken.sort(key=lambda a: a["name"])
+    today = datetime.date.today().isoformat()
+
+    lines = [
+        f"Automated URL health check from {today}.",
+        "",
+        f"Checked {len(apps)} catalog download URLs, found {len(broken)} broken.",
+        "",
+    ]
+    if broken:
+        lines += [
+            "These apps cannot currently be uploaded to Intune. Either the vendor "
+            "moved or removed the file (wait for the Homebrew cask to catch up) or "
+            "the app is discontinued and its JSON should be flagged with "
+            '"deprecated": true.',
+            "",
+            "| App | Version | Problem | URL |",
+            "| --- | --- | --- | --- |",
+        ]
+        for app in broken:
+            lines.append(
+                f"| {app['name']} | {app['version']} | {app['verdict']}: {app['detail']} | {app['url']} |"
+            )
+    else:
+        lines.append("All download URLs are healthy.")
+
+    with open(REPORT_FILE, "w") as f:
+        f.write("\n".join(lines) + "\n")
+
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a") as f:
+            f.write(f"broken_count={len(broken)}\n")
+
+    print(f"\n{len(broken)} of {len(apps)} URLs are broken. Report written to {REPORT_FILE}.")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/collect_app_info.py

@@ -1430,10 +1430,43 @@ def find_bundle_id(json_string):
 
     return None
 
+class CaskUnavailableError(Exception):
+    """Raised when a Homebrew cask is deprecated or disabled upstream."""
+    def __init__(self, display_name, reason):
+        super().__init__(f"{display_name}: {reason}")
+        self.display_name = display_name
+        self.reason = reason
+
+def mark_app_deprecated(apps_folder, display_name, reason):
+    """Flag an existing app JSON as deprecated so it is excluded from supported_apps.json."""
+    file_path = os.path.join(apps_folder, f"{sanitize_filename(display_name)}.json")
+    if not os.path.exists(file_path):
+        print(f"Cask for {display_name} is unavailable ({reason}) and has no local JSON file, skipping")
+        return
+    with open(file_path, "r") as f:
+        app_data = json.load(f)
+    if app_data.get("deprecated") and app_data.get("deprecation_reason") == reason:
+        print(f"{display_name} is already flagged as deprecated")
+        return
+    app_data["deprecated"] = True
+    app_data["deprecation_reason"] = reason
+    with open(file_path, "w") as f:
+        json.dump(app_data, f, indent=2)
+    print(f"Flagged {display_name} as deprecated: {reason}")
+
 def get_homebrew_app_info(json_url, needs_packaging=False, is_pkg_in_dmg=False, is_pkg_in_pkg=False, is_pkg=False):
     response = requests.get(json_url)
     response.raise_for_status()
     data = response.json()
+
+    # A deprecated or disabled cask means the vendor discontinued the app or
+    # its download can no longer be fetched reliably. Its URL will rot, so it
+    # must not be offered for upload.
+    if data.get("deprecated") or data.get("disabled"):
+        status = "disabled" if data.get("disabled") else "deprecated"
+        reason = data.get("disable_reason") or data.get("deprecation_reason") or "no reason given"
+        raise CaskUnavailableError(data["name"][0], f"{status} in Homebrew: {reason}")
+
     json_string = json.dumps(data)
 
     bundle_id = find_bundle_id(json_string)
@@ -1694,6 +1727,10 @@ def main():
                     # previous_version always equals version (noted in Issue #116)
                     existing_data["previous_version"] = existing_data.get("version", "")
 
+                    # The cask is healthy again, so drop any stale deprecation flag
+                    existing_data.pop("deprecated", None)
+                    existing_data.pop("deprecation_reason", None)
+
                     # Always update version and url
                     existing_data["version"] = new_version
                     existing_data["url"] = app_info["url"]
@@ -1727,6 +1764,8 @@ def main():
 
             apps_info.append(app_info)
             print(f"Saved app information for {display_name} to {file_path}")
+        except CaskUnavailableError as e:
+            mark_app_deprecated(apps_folder, e.display_name, e.reason)
         except Exception as e:
             print(f"Error processing special app {url}: {str(e)}")
             print(f"Full error details: ", e)
@@ -1775,7 +1814,7 @@ def main():
 
                     # Preserve all existing data except version, url, sha, and previous_version
                     for key in existing_data:
-                        if key not in ["version", "url", "sha", "previous_version"]:
+                        if key not in ["version", "url", "sha", "previous_version", "deprecated", "deprecation_reason"]:
                             app_info[key] = existing_data[key]
 
                     # Update version, url, sha and previous_version
@@ -1798,6 +1837,8 @@ def main():
 
             apps_info.append(app_info)
             print(f"Saved app information for {display_name} to {file_path}")
+        except CaskUnavailableError as e:
+            mark_app_deprecated(apps_folder, e.display_name, e.reason)
         except Exception as e:
             print(f"Error processing {url}: {str(e)}")
 
@@ -1822,7 +1863,7 @@ def main():
 
                     # Preserve all existing data except version, url and previous_version
                     for key in existing_data:
-                        if key not in ["version", "url", "previous_version"]:
+                        if key not in ["version", "url", "previous_version", "deprecated", "deprecation_reason"]:
                             app_info[key] = existing_data[key]
 
                     # Update version, url and previous_version
@@ -1844,6 +1885,8 @@ def main():
 
             apps_info.append(app_info)
             print(f"Saved app information for {display_name} to {file_path}")
+        except CaskUnavailableError as e:
+            mark_app_deprecated(apps_folder, e.display_name, e.reason)
         except Exception as e:
             print(f"Error processing PKG in PKG app {url}: {str(e)}")
 
@@ -1892,7 +1935,7 @@ def main():
 
                     # Preserve all existing data except version, url, sha and previous_version
                     for key in existing_data:
-                        if key not in ["version", "url", "sha", "previous_version"]:
+                        if key not in ["version", "url", "sha", "previous_version", "deprecated", "deprecation_reason"]:
                             app_info[key] = existing_data[key]
 
                     # Update version, url, sha and previous_version
@@ -1917,6 +1960,8 @@ def main():
 
             apps_info.append(app_info)
             print(f"Saved app information for {display_name} to {file_path}")
+        except CaskUnavailableError as e:
+            mark_app_deprecated(apps_folder, e.display_name, e.reason)
         except Exception as e:
             print(f"Error processing direct PKG app {url}: {str(e)}")
 
@@ -1941,7 +1986,7 @@ def main():
 
                     # Preserve all existing data except version, url and previous_version
                     for key in existing_data:
-                        if key not in ["version", "url", "previous_version"]:
+                        if key not in ["version", "url", "previous_version", "deprecated", "deprecation_reason"]:
                             app_info[key] = existing_data[key]
 
                     # Update version, url and previous_version
@@ -1962,6 +2007,8 @@ def main():
 
             apps_info.append(app_info)
             print(f"Saved app information for {display_name} to {file_path}")
+        except CaskUnavailableError as e:
+            mark_app_deprecated(apps_folder, e.display_name, e.reason)
         except Exception as e:
             print(f"Error processing PKG in DMG app {url}: {str(e)}")
 

.github/workflows/build-app-packages.yml

@@ -47,6 +47,10 @@ jobs:
           apps_to_build=()
           for file in Apps/*.json; do
             if [ -f "$file" ]; then
+              if [ "$(jq -r '.deprecated // false' "$file")" = "true" ]; then
+                echo "Skipping deprecated app: $(basename "$file" .json)"
+                continue
+              fi
               type=$(jq -r '.type // empty' "$file")
               if [ "$type" = "app" ] || [ "$type" = "pkg_in_dmg" ] || [ "$type" = "pkg_in_pkg" ] || [ "$type" = "pkg" ]; then
                 app_name=$(basename "$file" .json)
@@ -70,6 +74,16 @@ jobs:
 
           for filename in os.listdir(apps_folder):
               if filename.endswith(".json"):
+                  # Deprecated apps stay in Apps/ for history but must not be
+                  # offered for upload, their download URLs no longer work
+                  try:
+                      with open(os.path.join(apps_folder, filename)) as f:
+                          app_data = json.load(f)
+                  except ValueError:
+                      app_data = {}
+                  if app_data.get("deprecated"):
+                      print(f"Excluding deprecated app: {filename}")
+                      continue
                   app_name = os.path.splitext(filename)[0]
                   supported_apps[app_name] = f"https://raw.githubusercontent.com/ugurkocde/IntuneBrew/main/Apps/{filename}"
 

.github/workflows/url-health-check.yml

@@ -0,0 +1,73 @@
+name: Check Download URL Health
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * 1,4" # Mondays and Thursdays at 06:00 UTC
+
+permissions:
+  contents: read
+  issues: write
+
+env:
+  ISSUE_TITLE: Broken download URLs in app catalog
+
+jobs:
+  check-urls:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install dependencies
+        run: pip install requests
+
+      - name: Check download URLs
+        id: check
+        run: python .github/scripts/check_download_urls.py
+
+      - name: Find existing report issue
+        id: issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \
+            --search "\"$ISSUE_TITLE\" in:title" --json number,title \
+            --jq ".[] | select(.title == \"$ISSUE_TITLE\") | .number" | head -1)
+          echo "number=$number" >> "$GITHUB_OUTPUT"
+
+      - name: Create or update report issue
+        if: steps.check.outputs.broken_count != '0'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ISSUE_NUMBER: ${{ steps.issue.outputs.number }}
+        run: |
+          if [ -n "$ISSUE_NUMBER" ]; then
+            gh issue edit "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" --body-file url-health-report.md
+            echo "Updated issue #$ISSUE_NUMBER"
+          else
+            gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file url-health-report.md
+          fi
+
+      - name: Close report issue when everything is healthy
+        if: steps.check.outputs.broken_count == '0' && steps.issue.outputs.number != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ISSUE_NUMBER: ${{ steps.issue.outputs.number }}
+        run: |
+          gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
+            --comment "All catalog download URLs are healthy again."
+
+      - name: Upload report artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: url-health-report
+          path: url-health-report.md
+          if-no-files-found: ignore

Apps/archi.json

@@ -9,5 +9,7 @@
   "fileName": "Archi-Mac-Silicon-5.7.0.dmg",
   "category": "Design",
   "publisher": "The Archi Team",
-  "previous_version": "5.7.0"
+  "previous_version": "5.7.0",
+  "deprecated": true,
+  "deprecation_reason": "disabled in Homebrew: the developer intentionally makes distribution difficult for package managers"
 }