c.com Link checker fix

.github/workflows/live-links.yaml

@@ -8,60 +8,19 @@ jobs:
   check-links:
     if: github.repository == 'canonical/canonical.com'
     runs-on: ubuntu-latest
+    timeout-minutes: 60
 
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
       - name: Install linkchecker
         run: sudo pip install LinkChecker
 
-      - name: Write linkchecker config file
-        run: |
-          mkdir -p ~/.linkchecker
-          cat > ~/.linkchecker/linkcheckerrc <<EOF
-          [checking]
-          maxrequestspersecond=5
-          recursionlevel=2
-          timeout=1000
-          sslverify=0
-
-          [filtering]
-          checkextern=1
-          ignore=
-            https://res.cloudinary.com
-            q_auto
-            fl_sanitize
-            c_fill
-            e_sharpen
-            w_[0-9]*
-            h_[0-9]*
-            https://canonical.com/blog
-            https://ubuntu.com/blog
-            https://canonical.com/static/css/*
-            https://www.xilinx.com/*
-            https://player.vimeo.com/video/*
-            http://atea.com
-            https://linuxpolska.com/*
-            https://www.packet.net
-            https://www.amd.com/
-            https://start.microsemi.com
-            http://www.omnicore-it.com/
-            http://www.pgp.com/
-            https://www.hcl.com/
-            http://www.randrinc.com/
-            https://www.storagemadeeasy.com/
-
-          [output]
-          status=0
-          warnings=0
-          ignoreerrors=
-            ^http?s://.* ^.*(471|500|503|504|400|403|401)
-            ^http?s://.* Connection aborted
-            ^http?s://.* Connection reset by peer
-            ^http?s://.* RemoteDisconnected
-            ^http?s://.* Read timed out
-          EOF
-
-      - name: Run linkchecker for 404 errors only
-        run: linkchecker --no-warning https://canonical.com
+      - name: Run linkchecker with retry for transient failures
+        run: bash scripts/check-links.sh https://canonical.com
 
       - name: Send message on failure
         if: failure()

.github/workflows/pr.yaml

@@ -182,3 +182,14 @@
           github-token: ${{ secrets.GITHUB_TOKEN }}
           reporter: github-pr-check
           fail-on-error: true
+
+  test-shell:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Run shell tests
+        run: yarn test-shell

.linkchecker/linkcheckerrc

@@ -0,0 +1,41 @@
+[checking]
+maxrequestspersecond=5
+recursionlevel=2
+timeout=60
+sslverify=0
+
+[filtering]
+checkextern=1
+ignore=
+  https://res.cloudinary.com
+  q_auto
+  fl_sanitize
+  c_fill
+  e_sharpen
+  w_[0-9]*
+  h_[0-9]*
+  https://canonical.com/blog
+  https://ubuntu.com/blog
+  https://canonical.com/static/css/*
+  https://www.xilinx.com/*
+  https://player.vimeo.com/video/*
+  http://atea.com
+  https://linuxpolska.com/*
+  https://www.packet.net
+  https://www.amd.com/
+  https://start.microsemi.com
+  http://www.omnicore-it.com/
+  http://www.pgp.com/
+  https://www.hcl.com/
+  http://www.randrinc.com/
+  https://www.storagemadeeasy.com/
+
+[output]
+status=0
+warnings=0
+ignoreerrors=
+  ^https?://.* ^.*(471|500|503|504|400|403|401)
+  ^https?://.* Connection aborted
+  ^https?://.* Connection reset by peer
+  ^https?://.* RemoteDisconnected
+  ^https?://.* Read timed out

package.json

@@ -15,12 +15,13 @@
     "lint-llms": "python3 webapp/llms.py lint",
     "serve": "./entrypoint 0.0.0.0:${PORT}",
     "start": "yarn run build && concurrently --raw 'yarn run watch' 'yarn run serve'",
-    "test": "yarn run lint-scss && yarn run lint-python && yarn run lint-html && yarn run lint-llms && yarn run test-python",
+    "test": "yarn run lint-scss && yarn run lint-python && yarn run lint-html && yarn run lint-llms && yarn run test-python && yarn run test-shell",
     "test-python": "python3 -m unittest discover tests",
     "test-js": "jest --env=jsdom",
     "test-marketo": "python3 -m unittest tests.test_marketo",
     "test-kh": "python3 -m unittest tests.test_views.TestKnowledgeSections",
     "test-e2e": "playwright test",
+    "test-shell": "for f in tests/shell/*.sh; do bash \"$f\" || exit 1; done",
     "percy-snapshot": "percy snapshot snapshots.js"
   },
   "dependencies": {

scripts/check-links.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# check-links.sh — run LinkChecker with retry logic for transient failures.
+#
+# Usage:
+#   ./scripts/check-links.sh [URL]
+#
+# Defaults to https://canonical.com when no URL is given.
+# Uses .linkchecker/linkcheckerrc from the repo root for config.
+#
+# Examples:
+#   ./scripts/check-links.sh                        # check live site
+#   ./scripts/check-links.sh http://localhost:8002  # check local dev server
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+TARGET_URL="${1:-https://canonical.com}"
+
+runtime_error() {
+  echo "LinkChecker failed unexpectedly. See the output above for details." >&2
+  exit 2
+}
+
+# Print the unique failed URLs (valid=False) from a linkchecker CSV.
+extract_failed() {
+  python3 - "$1" <<'PY'
+import csv
+import sys
+
+seen = set()
+with open(sys.argv[1], newline="") as f:
+    rows = (line for line in f if not line.startswith("#"))
+    for row in csv.DictReader(rows, delimiter=";"):
+        if row.get("valid", "").strip().lower() == "false":
+            url = row.get("urlname")
+            if url and url not in seen:
+                seen.add(url)
+                print(url)
+PY
+}
+
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+if linkchecker \
+    --config "${REPO_ROOT}/.linkchecker/linkcheckerrc" \
+    --no-warning \
+    -F "csv/${WORK_DIR}/failed-links.csv" \
+    "${TARGET_URL}"; then
+  echo "No broken links found."
+  exit 0
+else
+  checker_status=$?
+fi
+
+[ "$checker_status" -eq 1 ] || runtime_error
+extract_failed "${WORK_DIR}/failed-links.csv" > "${WORK_DIR}/current-urls.txt" || runtime_error
+[ -s "${WORK_DIR}/current-urls.txt" ] || runtime_error
+
+attempt=1
+max_attempts=3
+delay=60
+
+# Re-check only the previously failed links
+while [ "$attempt" -le "$max_attempts" ] && [ -s "${WORK_DIR}/current-urls.txt" ]; do
+  echo "Attempt $attempt/$max_attempts: rechecking $(wc -l < "${WORK_DIR}/current-urls.txt") link(s) after ${delay}s"
+  sleep "$delay"
+
+  mapfile -t urls < "${WORK_DIR}/current-urls.txt"
+  retry_csv="${WORK_DIR}/retry-${attempt}.csv"
+  if linkchecker \
+      --config "${REPO_ROOT}/.linkchecker/linkcheckerrc" \
+      --no-warning \
+      --recursion-level=0 \
+      --timeout=30 \
+      -F "csv/${retry_csv}" \
+      "${urls[@]}"; then
+    echo "All remaining links resolved on attempt $attempt."
+    exit 0
+  else
+    checker_status=$?
+  fi
+
+  [ "$checker_status" -eq 1 ] || runtime_error
+  extract_failed "$retry_csv" > "${WORK_DIR}/next-urls.txt" || runtime_error
+  [ -s "${WORK_DIR}/next-urls.txt" ] || runtime_error
+  mv "${WORK_DIR}/next-urls.txt" "${WORK_DIR}/current-urls.txt"
+  attempt=$((attempt + 1))
+  delay=$((delay * 2))
+done
+
+echo "Links still broken after $max_attempts attempts:"
+cat "${WORK_DIR}/current-urls.txt"
+exit 1

tests/shell/test-check-links.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# Smoke test for scripts/check-links.sh.
+#
+# Puts a fake `linkchecker` and a no-op `sleep` on PATH, so the real network and
+# the retry backoff are never hit. Run with: bash tests/shell/test-check-links.sh
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+TARGET="${REPO_ROOT}/scripts/check-links.sh"
+
+STUB_DIR="$(mktemp -d)"
+trap 'rm -rf "$STUB_DIR"' EXIT
+
+# STUB_EXITS: space-separated exit code per invocation.
+# STUB_FAILS: space-separated group per invocation; comma-separated failing
+#             URLs within a group, or "-" for none.
+cat > "${STUB_DIR}/linkchecker" <<'STUB'
+#!/usr/bin/env bash
+out=""
+prev=""
+for arg in "$@"; do
+  [ "$prev" = "-F" ] && out="${arg#csv/}"
+  prev="$arg"
+done
+
+n=$(( $(cat "$STUB_STATE") + 1 ))
+echo "$n" > "$STUB_STATE"
+printf '%s\n' "$*" >> "$STUB_LOG"
+
+read -ra exits <<< "$STUB_EXITS"
+read -ra groups <<< "$STUB_FAILS"
+code="${exits[$((n - 1))]:-0}"
+fails="${groups[$((n - 1))]:--}"
+
+if [ -n "$out" ]; then
+  {
+    echo "# generated by fake linkchecker"
+    echo "urlname;parentname;valid"
+    if [ "$fails" != "-" ]; then
+      IFS=',' read -ra urls <<< "$fails"
+      for u in "${urls[@]}"; do echo "${u};https://example.com;False"; done
+    fi
+    echo "https://example.com/ok;https://example.com;True"
+  } > "$out"
+fi
+
+exit "$code"
+STUB
+chmod +x "${STUB_DIR}/linkchecker"
+
+printf '#!/usr/bin/env bash\nexit 0\n' > "${STUB_DIR}/sleep"
+chmod +x "${STUB_DIR}/sleep"
+
+export STUB_STATE="${STUB_DIR}/count"
+export STUB_LOG="${STUB_DIR}/log"
+
+failures=0
+status=0
+output=""
+
+run_case() {
+  export STUB_EXITS="$1"
+  export STUB_FAILS="$2"
+  echo 0 > "$STUB_STATE"
+  : > "$STUB_LOG"
+  output="$(PATH="${STUB_DIR}:${PATH}" bash "$TARGET" https://example.com 2>&1)"
+  status=$?
+}
+
+check() {
+  local name="$1" expected="$2" actual="$3"
+  if [ "$expected" = "$actual" ]; then
+    echo "ok   - ${name}"
+  else
+    echo "FAIL - ${name}: expected '${expected}', got '${actual}'"
+    printf '%s\n' "$output" | sed 's/^/       | /'
+    failures=$((failures + 1))
+  fi
+}
+
+contains() {
+  local name="$1" needle="$2"
+  case "$output" in
+    *"$needle"*) echo "ok   - ${name}" ;;
+    *)
+      echo "FAIL - ${name}: output missing '${needle}'"
+      printf '%s\n' "$output" | sed 's/^/       | /'
+      failures=$((failures + 1))
+      ;;
+  esac
+}
+
+# 1. Clean run.
+run_case "0" "-"
+check "clean run exits 0" 0 "$status"
+contains "clean run reports success" "No broken links found."
+
+# 2. Transient failure that clears on the first retry.
+run_case "1 0" "https://example.com/flaky -"
+check "transient failure exits 0" 0 "$status"
+contains "transient failure reports recovery" "resolved on attempt 1"
+
+# 3. Persistent failure: initial run + 3 retries, then give up.
+run_case "1 1 1 1" "https://example.com/dead https://example.com/dead https://example.com/dead https://example.com/dead"
+check "persistent failure exits 1" 1 "$status"
+contains "persistent failure lists the URL" "https://example.com/dead"
+check "persistent failure uses all attempts" 4 "$(cat "$STUB_STATE")"
+
+# 4. Unexpected linkchecker exit code is not treated as a broken link.
+run_case "3" "-"
+check "runtime error exits 2" 2 "$status"
+contains "runtime error is reported" "LinkChecker failed unexpectedly"
+
+# 5. Duplicate failures are de-duplicated before the recheck.
+run_case "1 0" "https://example.com/a,https://example.com/a,https://example.com/b -"
+check "dedup run exits 0" 0 "$status"
+check "recheck receives 1 copy of /a" 1 "$(grep -c -o 'https://example.com/a' <<< "$(sed -n 2p "$STUB_LOG")")"
+contains "recheck reports 2 links" "rechecking 2 link(s)"
+
+echo
+if [ "$failures" -eq 0 ]; then
+  echo "All check-links.sh tests passed."
+else
+  echo "${failures} test(s) failed."
+fi
+exit $(( failures > 0 ))

webapp/handlers.py

@@ -193,6 +193,7 @@ def _fetch_google_supported_domains():
     ],
     "frame-src": [
         "'self'",
+        "assets.ubuntu.com",
         "*.doubleclick.net",
         "*.crazyegg.com",
         "www.youtube.com/",