Merge branch 'main' into data_platform_docs_links

Author: izmalk

.github/workflows/copilot-assignment.lock.yml

@@ -1,5 +1,5 @@
 ---
-# GH agentic workflow frontmatter docs: 
+# GH agentic workflow frontmatter docs:
 # https://github.github.com/gh-aw/reference/frontmatter-full/#complete-frontmatter-reference
 # Triggered when the "copilot" label is added to an issue.
 # Only runs for users with write access (admin, maintainer, write roles).
@@ -10,9 +10,7 @@ on:
 
 if: github.event.label.name == 'copilot'
 
-engine:
-  id: copilot
-  model: gpt-5.5
+engine: copilot
 
 permissions:
   issues: read
@@ -24,7 +22,6 @@ safe-outputs:
   assign-to-agent:
     name: copilot
     target: triggering
-    model: gpt-5.5
 ---
 
 Assign the GitHub Copilot coding agent to this issue. The agent will read the issue, implement a solution, and open a pull request for review.

.github/workflows/copilot-assignment.md

@@ -69,6 +69,20 @@ jobs:
           yarn install
           yarn run build
 
+      - name: Pre-generate llms-full.txt
+        env:
+          SECRET_KEY: insecure_test_key
+          APPLICATION_CRYPTO_SECRET_KEY: insecure_test_key
+          HARVEST_API_KEY: local_development_fake_key
+          SERVICE_ACCOUNT_EMAIL: test_email@email.com
+          SERVICE_ACCOUNT_PRIVATE_KEY: test_private_key
+        run: |
+          python3 -m venv .venv-llms
+          .venv-llms/bin/pip install --upgrade pip "setuptools<81"
+          .venv-llms/bin/pip install -r requirements.txt
+          .venv-llms/bin/python webapp/llms.py generate
+          rm -rf .venv-llms
+
       - name: Setup LXD
         uses: canonical/setup-lxd@main
 
@@ -171,12 +185,12 @@ jobs:
         run: |
           sudo snap install juju --channel=3.6/stable --classic
           sudo snap install vault --classic
-
+  
       - name: Download Charm Artifact
         uses: actions/download-artifact@v4
         with:
           name: canonical-com-charm
-
+ 
       - name: Configure Vault and Juju
         run: |
           export VAULT_ADDR=https://vault.admin.canonical.com:8200

.github/workflows/deploy.yaml

@@ -72,6 +72,27 @@ jobs:
       - name: Lint python
         run: yarn lint-python
 
+  lint-llms:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install node dependencies
+        run: yarn install --immutable
+
+      - name: Install python dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          sudo pip3 install pyyaml
+
+      - name: Lint llms.txt and llms.yaml
+        run: yarn lint-llms
+
   lint-jinja:
     runs-on: ubuntu-22.04
 

.github/workflows/pr.yaml

@@ -69,6 +69,7 @@ static/js/modules/
 static/js/dist/
 *.swp
 templates/sitemap_tree.xml
+templates/llms-full.txt
 
 # Playwright
 /test-results/

.gitignore

@@ -22,6 +22,8 @@
 - `templates/sitemap-index.xml` - top-level sitemap index that links to all section sitemaps.
 - `templates/sitemap-links.xml`, `templates/careers/sitemap.xml`, `templates/partners/sitemap.xml`, `templates/knowledge/sitemap.xml`, `templates/data/sitemap.xml` - per-section sitemap templates rendered by routes in `webapp/app.py`.
 - `templates/sitemap_tree.xml` - generated (git-ignored) full sitemap; the `/sitemap_tree.xml` route (`build_sitemap_tree` in `webapp/app.py`) regenerates it from the `templates/` tree on a `POST` and serves it on `GET`.
+- `templates/llms.txt` - manually maintained LLM-friendly site index (https://llmstxt.org/), committed to git (not generated). Edit this file directly to change what AI crawlers see. Combined with `llms.yaml` at app startup (`webapp/llms.py`, `build_llms_txt`) and served by the `/llms.txt` route (`llms_txt` in `webapp/app.py`).
+- `templates/llms-full.txt` - generated (git-ignored) full Markdown content of every renderable page linked from `llms.txt` (`webapp/llms.py`, `build_llms_full_txt`). Pre-generated at build time by `python3 webapp/llms.py generate` (run in the `pack-rock` job of `.github/workflows/deploy.yaml` before `rockcraft pack`); the `/llms-full.txt` route (`llms_full_txt` in `webapp/app.py`) serves it read-only from disk, regenerating on demand only if missing (local dev). Deliberately skips pages served live from Discourse (product docs, tutorials, case studies, events - see `DISCOURSE_PATH_PREFIXES` in `webapp/llms.py`), since the build environment has no Discourse API access and shouldn't scrape and redistribute that content.
 - `static/js/` - browser JS/TS entrypoints and React code.
   - `static/js/career-explorer/` - React/TypeScript careers explorer.
   - `static/js/canonical-cla/` - React client for the CLA pages.
@@ -32,6 +34,7 @@
 - `navigation.yaml` - primary navigation data source.
 - `secondary-navigation.yaml` - secondary nav/page sidebar data source.
 - `redirects.yaml` - redirects config.
+- `llms.yaml` - docs-team-owned config for extra curated links inserted into `/llms.txt` right after its "Main pages" section (`heading` + `links`, each with `title`/`url`/`description`), read by `webapp/llms.py`. Must be listed in `rockcraft.yaml` so it ships in the image. Formatting of both this file and `templates/llms.txt` is checked by `yarn lint-llms` (`python3 webapp/llms.py lint`), a required PR check (`lint-llms` job in `.github/workflows/pr.yaml`).
 - `tests/` - Python unit tests plus JS/Jest and Playwright tests.
   - `tests/js/` - Jest tests for browser modules.
   - `tests/playwright/` - E2E tests; `tests/playwright/tests/navigation/README.md` explains navigation coverage.

AGENTS.md

@@ -0,0 +1,28 @@
+# =============================================================================
+#  llms.yaml  —  extra curated links for /llms.txt
+# =============================================================================
+#
+#  WHAT IS THIS FILE?
+#  ------------------
+#  templates/llms.txt is the hand-written, canonical version of
+#  https://canonical.com/llms.txt (see https://llmstxt.org/). This file lets
+#  you add extra curated links to that page — for example, links to product
+#  documentation or community sites that live outside this repo — without
+#  editing templates/llms.txt directly.
+#
+#  Links are grouped into sections. Each section has:
+#      heading:   the section title (e.g. "Documentation")
+#      links:     a list of links, each with:
+#                     title:        the link text                 (required)
+#                     url:          the full web address          (required)
+#                     description:  one short sentence about it    (required)
+#
+#  EXAMPLE — this one is live. Copy a link block (the three "- title" lines)
+#  to add more, and add another "- heading" block to start a new section.
+#
+extra:
+  - heading: Documentation
+    links:
+      - title: Juju
+        url: https://canonical.com/juju/docs/juju-cli/page/llms.txt
+        description: Command reference and documentation for the Juju CLI.

llms.yaml

@@ -12,9 +12,10 @@
     "lint-python": "flake8 --extend-ignore=E203 webapp tests && black --check --line-length 79 webapp tests",
     "lint-scss": "stylelint static/**/*.scss",
     "lint-html": "djlint templates/ --lint",
+    "lint-llms": "python3 webapp/llms.py lint",
     "serve": "./entrypoint 0.0.0.0:${PORT}",
     "start": "yarn run build && concurrently --raw 'yarn run watch' 'yarn run serve'",
-    "test": "yarn run lint-scss && yarn run lint-python && yarn run lint-html && yarn run test-python",
+    "test": "yarn run lint-scss && yarn run lint-python && yarn run lint-html && yarn run lint-llms && yarn run test-python",
     "test-python": "python3 -m unittest discover tests",
     "test-js": "jest --env=jsdom",
     "test-marketo": "python3 -m unittest tests.test_marketo",

package.json

@@ -659,9 +659,8 @@ mir/doc/md__r_e_a_d_m_e.*: https://github.com/canonical/mir/blob/main/README.md
 mir/doc/.*: https://canonical-mir.readthedocs-hosted.com/stable/
 mir/tutorials: /mir/docs/tutorials/
 mir/tutorials/.*: /mir/docs/tutorials/
-legal/ubuntu-pro-description/?: "https://assets.ubuntu.com/v1/f744d687-ubuntu_pro_description_unified.pdf"
-legal/ubuntu-pro-description/ja?: "https://assets.ubuntu.com/v1/f744d687-ubuntu_pro_description_unified.pdf"
-legal/ubuntu-pro-description/jp?: "https://assets.ubuntu.com/v1/f744d687-ubuntu_pro_description_unified.pdf"
+legal/ubuntu-pro-description/ja?: "/legal/ubuntu-pro-description"
+legal/ubuntu-pro-description/jp?: "/legal/ubuntu-pro-description"
 legal/ubuntu-advantage-service-terms(?P<page>.*)/?: "/legal/ubuntu-pro-service-terms{page}"
 
 # Multipass documentation
@@ -2046,5 +2045,4 @@ blog/cloudify-your-data-centre: https://assets.ubuntu.com/v1/855af1ef-infrastruc
 # Hiring blog redirect
 blog/how-we-hire-at-canonical: /careers/hiring-process
 
-
-robots.txt?: "/static/files/robots.txt"
+robots.txt?: "/static/files/robots.txt"

redirects.yaml

@@ -8,7 +8,7 @@ canonicalwebteam.search==2.1.2
 canonicalwebteam.form-generator==2.2.0
 canonicalwebteam.directory-parser==1.2.11
 canonicalwebteam.markdown-response==0.2.0
-bleach==5.0.1
+bleach==6.4.0
 markdown==3.8.2
 python-slugify==8.0.4
 vcrpy-unittest==0.1.7