feat(apiref): add support for generating and validating API catalogs

Author: ziflex

.github/workflows/publish-core-api.yml

@@ -1,4 +1,4 @@
-name: Publish Ferret Core API Reference
+name: Publish Ferret Core API Artifacts
 
 on:
   release:
@@ -15,7 +15,7 @@ permissions:
 
 jobs:
   publish:
-    name: Publish versioned API Reference
+    name: Publish versioned API artifacts
     if: github.event_name == 'workflow_dispatch' || startsWith(github.event.release.tag_name, 'v2.')
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -66,13 +66,14 @@ jobs:
             tools/apiref/go.sum
             tools/apipublish/go.sum
 
-      - name: Generate Ferret Core API Reference
+      - name: Generate Ferret Core API artifacts
         env:
           API_VERSION: ${{ steps.release.outputs.version }}
         run: >-
           go -C tools/apiref run .
           -version "$API_VERSION"
           -o "$RUNNER_TEMP/montferret-core-api.json"
+          -catalog "$RUNNER_TEMP/montferret-core-catalog.json"
 
       - name: Check out existing Pages branch
         uses: actions/checkout@v4
@@ -81,12 +82,12 @@ jobs:
           path: pages
           fetch-depth: 0
 
-      - name: Publish and push immutable API Reference
+      - name: Publish and push immutable API artifacts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
         run: |
           set -euo pipefail
           git -C pages config user.name "github-actions[bot]"
           git -C pages config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          ./scripts/publish-core-api.sh "$RUNNER_TEMP/montferret-core-api.json" pages origin gh-pages
+          ./scripts/publish-core-api.sh "$RUNNER_TEMP/montferret-core-api.json" "$RUNNER_TEMP/montferret-core-catalog.json" pages origin gh-pages

docs/maintainers/core-api-reference.md

@@ -1,6 +1,6 @@
-# Ferret Core API Reference
+# Ferret Core API Artifacts
 
-Ferret publishes a versioned API Reference with the canonical identity
+Ferret publishes a versioned API Reference and sibling API Catalog with the canonical identity
 `montferret/core`. This identity describes optional built-in functionality
 provided by Ferret itself. It is not an installable Registry module and must not
 be added to a module manifest as a dependency.
@@ -24,7 +24,7 @@ lowercase presentation in the registry and generated API Reference. This include
 every namespace segment; casing compatibility is represented by lookup rather
 than duplicate aliases.
 
-The API Reference and discovery-index wire contracts belong to
+The API Reference, API Catalog, and discovery-index wire contracts belong to
 [`github.com/MontFerret/specs`](https://github.com/MontFerret/specs). Ferret pins
 the released Specs version in the independent generator and publisher modules,
 which validate every completed reference and index. Ferret's root module does
@@ -75,28 +75,34 @@ Run the generator from the repository root with an unprefixed canonical SemVer:
 ```sh
 GOWORK=off go -C tools/apiref run . \
   -version 2.0.0-alpha.45 \
-  -o /tmp/montferret-core-api.json
+  -o /tmp/montferret-core-api.json \
+  -catalog /tmp/montferret-core-catalog.json
 ```
 
-The command emits diagnostics only on stderr and atomically writes deterministic,
-two-space-indented JSON with one trailing newline. The generator contains no
+The command emits diagnostics only on stderr and writes deterministic,
+two-space-indented JSON with one trailing newline. `api.json` remains the
+canonical callable API. `catalog.json` contains presentation categories for
+global functions and real namespace roots. Categories such as `math` and
+`strings` are not callable Ferret namespaces. The generator contains no
 deployment domain.
 
 ## Release publication
 
-The `Publish Ferret Core API Reference` workflow runs independently from
+The `Publish Ferret Core API Artifacts` workflow runs independently from
 dependent-release notifications. It accepts `release.published` events and
 manual dispatches, verifies that the selected canonical v2 tag belongs to a
 published GitHub release, checks out that exact tag, strips the leading `v`, and
-generates the artifact.
+generates both artifacts.
 
 The existing `gh-pages` branch is updated through the repository-local
 `scripts/publish-core-api.sh` wrapper. Publication validates the existing index
 and every referenced artifact before mutation, preserves unrelated Pages files,
-creates one version artifact, recomputes the index, commits once, and performs a
+atomically installs one version directory containing `api.json` and
+`catalog.json`, recomputes the unchanged API index, commits once, and performs a
 normal non-force push. Version directories, index entries, and hrefs are
-immutable and cannot be republished even when bytes match. A stale push fails;
-published history is never rewritten.
+immutable and cannot be republished even when bytes match. Existing API-only
+version directories remain valid legacy state and are never backfilled. A stale
+push fails; published history is never rewritten.
 
 All `gh-pages` writers use the repository-wide `gh-pages-writer` concurrency key
 with cancellation disabled. Unit and integration benchmarks still execute in
@@ -110,6 +116,7 @@ The public documents are:
 
 - `https://ferretlang.org/ferret/index.json`
 - `https://ferretlang.org/ferret/versions/<version>/api.json`
+- `https://ferretlang.org/ferret/versions/<version>/catalog.json`
 
 These URLs share the existing Ferret Pages root, so benchmark history and other
 site files remain alongside the API Reference without being regenerated.

scripts/publish-core-api.sh

@@ -2,24 +2,31 @@
 
 set -eu
 
-if [ "$#" -lt 2 ] || [ "$#" -gt 4 ]; then
-	printf 'Usage: %s <api-reference> <gh-pages-checkout> [remote] [branch]\n' "$0" >&2
+if [ "$#" -lt 3 ] || [ "$#" -gt 5 ]; then
+	printf 'Usage: %s <api-reference> <api-catalog> <gh-pages-checkout> [remote] [branch]\n' "$0" >&2
 	exit 2
 fi
 
 script_dir=$(CDPATH='' cd -- "$(dirname "$0")" && pwd)
 source_root=$(CDPATH='' cd -- "$script_dir/.." && pwd)
 reference_dir=$(CDPATH='' cd -- "$(dirname "$1")" && pwd)
 reference="$reference_dir/$(basename "$1")"
-pages_root=$(CDPATH='' cd -- "$2" && pwd)
-remote=${3:-origin}
-branch=${4:-gh-pages}
+catalog_dir=$(CDPATH='' cd -- "$(dirname "$2")" && pwd)
+catalog="$catalog_dir/$(basename "$2")"
+pages_root=$(CDPATH='' cd -- "$3" && pwd)
+remote=${4:-origin}
+branch=${5:-gh-pages}
 
 if [ ! -f "$reference" ]; then
 	printf 'API Reference not found: %s\n' "$reference" >&2
 	exit 1
 fi
 
+if [ ! -f "$catalog" ]; then
+	printf 'API Catalog not found: %s\n' "$catalog" >&2
+	exit 1
+fi
+
 if ! git -C "$pages_root" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
 	printf 'Pages checkout is not a Git worktree: %s\n' "$pages_root" >&2
 	exit 1
@@ -34,15 +41,17 @@ version=$(jq -er '.version | strings | select(length > 0)' "$reference")
 
 go -C "$source_root/tools/apipublish" run . \
 	-reference "$reference" \
+	-catalog "$catalog" \
 	-pages "$pages_root"
 
-artifact="versions/$version/api.json"
-git -C "$pages_root" add -- index.json "$artifact"
+reference_artifact="versions/$version/api.json"
+catalog_artifact="versions/$version/catalog.json"
+git -C "$pages_root" add -- index.json "$reference_artifact" "$catalog_artifact"
 
 if git -C "$pages_root" diff --cached --quiet; then
 	printf 'Publication produced no staged changes for %s\n' "$version" >&2
 	exit 1
 fi
 
-git -C "$pages_root" commit -m "Publish montferret/core API $version" -- index.json "$artifact"
+git -C "$pages_root" commit -m "Publish montferret/core API artifacts $version" -- index.json "$reference_artifact" "$catalog_artifact"
 git -C "$pages_root" push "$remote" "HEAD:refs/heads/$branch"

scripts/publish_core_api_test.go

@@ -37,27 +37,32 @@ func TestPublishCoreAPIRejectsStaleNonFastForwardPush(t *testing.T) {
 
 	firstReference := filepath.Join(temporary, "first-api.json")
 	secondReference := filepath.Join(temporary, "second-api.json")
+	firstCatalog := filepath.Join(temporary, "first-catalog.json")
+	secondCatalog := filepath.Join(temporary, "second-catalog.json")
 	writeReference(t, firstReference, "2.0.0-alpha.1")
 	writeReference(t, secondReference, "2.0.0-alpha.2")
+	writeCatalog(t, firstCatalog, "2.0.0-alpha.1")
+	writeCatalog(t, secondCatalog, "2.0.0-alpha.2")
 
 	publish := filepath.Join(root, "scripts", "publish-core-api.sh")
-	runPublish(t, root, publish, firstReference, first, remote, true)
+	runPublish(t, root, publish, firstReference, firstCatalog, first, remote, true)
 	remoteAfterFirst := strings.TrimSpace(git(t, temporary, "--git-dir", remote, "rev-parse", "refs/heads/gh-pages"))
 
-	runPublish(t, root, publish, secondReference, second, remote, false)
+	runPublish(t, root, publish, secondReference, secondCatalog, second, remote, false)
 	remoteAfterStale := strings.TrimSpace(git(t, temporary, "--git-dir", remote, "rev-parse", "refs/heads/gh-pages"))
 	if remoteAfterStale != remoteAfterFirst {
 		t.Fatalf("stale push changed published history: before=%s after=%s", remoteAfterFirst, remoteAfterStale)
 	}
 
 	assertContent(t, filepath.Join(first, ".gitignore"), "public\n")
 	assertContent(t, filepath.Join(first, "versions", "2.0.0-alpha.1", "api.json"), string(read(t, firstReference)))
+	assertContent(t, filepath.Join(first, "versions", "2.0.0-alpha.1", "catalog.json"), string(read(t, firstCatalog)))
 }
 
-func runPublish(t *testing.T, root, script, reference, pages, remote string, wantSuccess bool) {
+func runPublish(t *testing.T, root, script, reference, catalog, pages, remote string, wantSuccess bool) {
 	t.Helper()
 
-	command := exec.Command(script, reference, pages, remote, "gh-pages")
+	command := exec.Command(script, reference, catalog, pages, remote, "gh-pages")
 	command.Dir = root
 	command.Env = append(os.Environ(), "GOWORK=off")
 	output, err := command.CombinedOutput()
@@ -74,6 +79,32 @@ func runPublish(t *testing.T, root, script, reference, pages, remote string, wan
 	}
 }
 
+func writeCatalog(t *testing.T, path, version string) {
+	t.Helper()
+
+	catalog := map[string]any{
+		"schemaVersion": 1,
+		"id":            "montferret/core",
+		"version":       version,
+		"categories": []any{map[string]any{
+			"id":          "utils",
+			"title":       "Utilities",
+			"description": "General utility functions.",
+			"functions":   []string{"PING"},
+		}},
+		"namespaceRoots": []string{},
+	}
+
+	data, err := json.MarshalIndent(catalog, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func writeReference(t *testing.T, path, version string) {
 	t.Helper()
 

scripts/workflow_contract_test.go

@@ -44,6 +44,7 @@ func TestReleasePublicationChecksPublishedExactTagAndUsesNormalPushWrapper(t *te
 		"ref: refs/tags/${{ steps.release.outputs.tag }}",
 		"contents: write",
 		"go -C tools/apiref run .",
+		`-catalog "$RUNNER_TEMP/montferret-core-catalog.json"`,
 		"./scripts/publish-core-api.sh",
 	} {
 		if !strings.Contains(workflow, required) {

tools/apipublish/go.mod

@@ -4,7 +4,7 @@ go 1.25.0
 
 require (
 	github.com/Masterminds/semver/v3 v3.5.0
-	github.com/MontFerret/specs v1.9.0
+	github.com/MontFerret/specs v1.10.0
 )
 
 require (

tools/apipublish/go.sum

@@ -1,7 +1,7 @@
 github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
 github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
-github.com/MontFerret/specs v1.9.0 h1:IcqUfyxd/9R2TBIgeS499noPMmSptJp0cqrUAPmhcac=
-github.com/MontFerret/specs v1.9.0/go.mod h1:ENGyEPFjrWVPWOUAsBMc1acUsqpswieTrAJfhcm576E=
+github.com/MontFerret/specs v1.10.0 h1:T3p90YO7KNU0ROaNlJeoCZ2TBfibJfOiLEr1IOpzwo4=
+github.com/MontFerret/specs v1.10.0/go.mod h1:ENGyEPFjrWVPWOUAsBMc1acUsqpswieTrAJfhcm576E=
 github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
 github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=

tools/apipublish/internal/publisher/catalog.go

@@ -0,0 +1,73 @@
+package publisher
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/MontFerret/specs/pkg/api"
+	apicatalog "github.com/MontFerret/specs/pkg/api/catalog"
+)
+
+func validatePair(reference *api.Reference, catalog *apicatalog.Catalog) error {
+	problems := make([]error, 0)
+	if catalog.ID != reference.ID {
+		problems = append(problems, fmt.Errorf("catalog id %q does not match API id %q", catalog.ID, reference.ID))
+	}
+
+	if catalog.Version != reference.Version {
+		problems = append(problems, fmt.Errorf("catalog version %q does not match API version %q", catalog.Version, reference.Version))
+	}
+
+	globalFunctions := make(map[string]struct{})
+	namespaceRoots := make(map[string]struct{})
+	for _, namespace := range reference.Namespaces {
+		if namespace.Name == "" {
+			for _, function := range namespace.Functions {
+				globalFunctions[function.Name] = struct{}{}
+			}
+
+			continue
+		}
+
+		root, _, _ := strings.Cut(namespace.Name, "::")
+		namespaceRoots[root] = struct{}{}
+	}
+
+	categorized := make(map[string]string)
+	for _, category := range catalog.Categories {
+		for _, function := range category.Functions {
+			if _, exists := globalFunctions[function]; !exists {
+				problems = append(problems, fmt.Errorf("catalog category %q references unknown global function %q", category.ID, function))
+			}
+
+			if previous, exists := categorized[function]; exists {
+				problems = append(problems, fmt.Errorf("global function %q is assigned to categories %q and %q", function, previous, category.ID))
+			}
+
+			categorized[function] = category.ID
+		}
+	}
+
+	for function := range globalFunctions {
+		if _, exists := categorized[function]; !exists {
+			problems = append(problems, fmt.Errorf("global function %q is not assigned to a catalog category", function))
+		}
+	}
+
+	declaredRoots := make(map[string]struct{}, len(catalog.NamespaceRoots))
+	for _, root := range catalog.NamespaceRoots {
+		declaredRoots[root] = struct{}{}
+		if _, exists := namespaceRoots[root]; !exists {
+			problems = append(problems, fmt.Errorf("catalog namespace root %q does not cover an API namespace", root))
+		}
+	}
+
+	for root := range namespaceRoots {
+		if _, exists := declaredRoots[root]; !exists {
+			problems = append(problems, fmt.Errorf("API namespace root %q is not declared by the catalog", root))
+		}
+	}
+
+	return errors.Join(problems...)
+}