Merge branch 'main' into dependabot/go_modules/kubernetes-71c9e1377e

Author: mchmarny

.github/workflows/rekor-monitor.yaml

@@ -12,17 +12,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Transparency-log monitoring for AICR's release supply chain.
+# Transparency-log monitoring for AICR's release supply chain, on Rekor v2.
 #
-# Hourly, this calls the upstream sigstore/rekor-monitor reusable workflow to:
-#   - Consistency: verify the public-good Rekor log stays append-only between
-#     runs (checkpoint persisted as the `checkpoint` artifact).
-#   - Identity: scan entries added since the last run for AICR's release signing
-#     identity. An entry under that identity that a release did not produce
-#     signals OIDC/key compromise.
+# Hourly, this calls the upstream sigstore/rekor-monitor reusable workflow to do
+# both checks in a single job (see internal/cmd MonitorLoop upstream):
+#   - Consistency: prove the Rekor v2 log stays append-only between runs (Merkle
+#     consistency from the last checkpoint to the current tree head). O(log n),
+#     always finishes in seconds. Checkpoint persisted as the `checkpoint`
+#     artifact.
+#   - Identity: scan entries added since the last checkpoint for AICR's release
+#     signing identity. An entry under that identity that a release did not
+#     produce signals OIDC/key compromise.
 # On any failure (consistency break or matched identity), it files an issue
 # (`file_issue: true`).
 #
+# Why v2 (NVIDIA/aicr#1623). Identity monitoring is a linear scan of every entry
+# added since the last checkpoint (Rekor's index cannot be queried by
+# certificate SAN, and our keyless release identity has no email or fixed key).
+# On the Rekor **v1** firehose that scan runs ~50x slower than the log grows, so
+# it can never keep up in a bounded CI job: the earlier v1 config timed out
+# every run and never completed a single scan. Rekor **v2** is tile-based: bulk
+# 256-entry reads make a single-worker scan outpace the log, so identity
+# monitoring becomes one cheap job. This rides on release signing having moved to
+# v2 in NVIDIA/aicr#1650 (only entries actually in v2 can be watched there).
+#
+# Selecting v2. The monitor picks its Rekor API version by matching its `url`
+# against the services in the TUF-distributed Sigstore SigningConfig; a match on
+# a v2 service switches it to v2. The resolve-v2-shard job computes that URL at
+# run time from the same signing config release signing uses (via `aicr trust
+# update --emit-signing-config`), so we never hardcode a shard and yearly shard
+# rotation (log2025-1 -> log2026-1 -> ...) needs no change here. Once v2 is
+# selected the *full* shard set is auto-discovered from the SigningConfig and
+# refreshed each run.
+#
 # The monitored identity is AICR's release signer (see .goreleaser.yaml and
 # .github/workflows/on-tag.yaml): the GitHub Actions OIDC SAN for on-tag.yaml,
 # issued by token.actions.githubusercontent.com.
@@ -52,8 +74,38 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  # Resolve the current Rekor v2 shard from the Sigstore signing config, so the
+  # monitor always selects v2 against a live shard without a hardcoded URL. We
+  # read the same TUF-distributed signing config that release signing resolves
+  # (`aicr trust update --emit-signing-config`, see pkg/trust), so the monitor
+  # provably watches where releases actually write, and yearly shard rotation
+  # (log2025-1 -> log2026-1 -> ...) needs no change here.
+  resolve-v2-shard:
+    name: Resolve current Rekor v2 shard
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read  # checkout to build the aicr CLI
+    outputs:
+      url: ${{ steps.resolve.outputs.url }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16  # v6.5.0
+        with:
+          go-version-file: go.mod
+      - id: resolve
+        name: Resolve v2 shard URL from the signing config
+        run: |
+          set -euo pipefail
+          go run ./cmd/aicr trust update --emit-signing-config signing-config.json
+          # Newest currently-listed v2 shard (majorApiVersion 2). Any live v2
+          # shard selects v2; the monitor then discovers the full set from TUF.
+          url="$(jq -er '[.rekorTlogUrls[] | select(.majorApiVersion == 2)] | sort_by(.validFor.start) | last | .url' signing-config.json)"
+          echo "Resolved Rekor v2 shard: $url"
+          echo "url=$url" >> "$GITHUB_OUTPUT"
+
   monitor:
-    name: AICR release identity + log consistency
+    name: AICR release identity + log consistency (Rekor v2)
+    needs: resolve-v2-shard
     permissions:
       contents: read  # checkout performed by the reusable workflow
       issues: write   # file an issue on monitoring failure
@@ -64,6 +116,10 @@ jobs:
       # Must exceed the hourly cron so the consistency checkpoint never expires
       # between runs; wide enough to survive a brief scheduling pause.
       artifact_retention_days: 30
+      # Rekor v2 shard resolved at run time from the signing config (see the
+      # resolve-v2-shard job). Selecting a v2 service switches the monitor to v2;
+      # it then auto-discovers the full shard set from TUF.
+      url: ${{ needs.resolve-v2-shard.outputs.url }}
       config: |
         monitoredValues:
           certIdentities:

docs/contributor/maintaining.md

@@ -53,17 +53,71 @@ failure during the first 60s after tag publish. Re-run the workflow.
 
 The `Rekor Monitor` workflow (`.github/workflows/rekor-monitor.yaml`) runs
 hourly and calls the upstream `sigstore/rekor-monitor` reusable workflow. It
-watches the public-good Rekor transparency log for two things: that the log
-stays append-only (consistency), and that no entry appears under AICR's release
-signing identity that a release did not produce (identity). On either failure it
-opens an issue.
+watches the **Rekor v2** transparency log (where AICR release signing writes
+since [#1650](https://github.com/NVIDIA/aicr/issues/1650)) for two things, both
+in one job: that the log stays append-only (consistency), and that no entry
+appears under AICR's release signing identity that a release did not produce
+(identity). On either failure it opens an issue.
 
 This protects the trust root every AICR consumer depends on: the release
 binaries, the signed recipe catalog, and the container images all chain to that
 one identity. When the workflow files an issue, follow the triage steps in the
 workflow file's header comment; an unrecognized identity hit should be treated
 as potential OIDC/key compromise.
 
+### Why v2, and why identity monitoring is feasible now
+
+Identity monitoring is a linear scan of every entry added to the log since the
+last checkpoint, because Rekor's index cannot be queried by certificate SAN and
+AICR's keyless release identity has no email or fixed public key to search on.
+On the Rekor **v1** firehose that scan runs roughly 50x slower than the log
+grows, so it can never keep up inside a bounded CI job: the earlier v1
+identity config timed out on every run and never completed a single scan
+([#1623](https://github.com/NVIDIA/aicr/issues/1623)). Rekor **v2** is
+tile-based: bulk 256-entry reads let a single worker outpace the log, so the
+identity scan is a cheap job that always finishes. This is why the whole design
+is a single unbounded scan again rather than sharded paging.
+
+### Shard selection (automatic, no manual re-tune)
+
+The monitor selects Rekor v2 by pointing its `url` at a v2 shard listed in the
+Sigstore `SigningConfig`; a match on a v2 service switches it to v2, after which
+it auto-discovers the **full** shard set from TUF and refreshes it every run.
+
+The shard URL is **not hardcoded**. The `resolve-v2-shard` job computes it at run
+time from the same TUF-distributed signing config that release signing resolves,
+then feeds it to the monitor:
+
+```bash
+# Same command the resolve-v2-shard job runs (works from a fresh checkout).
+go run ./cmd/aicr trust update --emit-signing-config signing-config.json
+jq -er '[.rekorTlogUrls[] | select(.majorApiVersion == 2)] | sort_by(.validFor.start) | last | .url' signing-config.json
+```
+
+Because it reads the signing config directly, the monitor provably watches where
+releases actually write, and yearly shard rotation (`log2025-1` -> `log2026-1`
+-> ...) needs no change to this workflow. If the resolve job ever fails (for
+example the TUF CDN is unreachable), the monitor job is skipped for that run and
+retries on the next hourly tick.
+
+### First run after switching from v1: reset the checkpoint
+
+The `checkpoint` artifact persists a **v1** checkpoint from the prior config; a
+v2 run cannot parse it and will fail. After merging a change that moves this
+workflow to v2, delete the stale artifact once so the first v2 run establishes a
+fresh v2 baseline (it saves the current v2 tree head and scans forward from
+there):
+
+```bash
+gh api "repos/NVIDIA/aicr/actions/artifacts?name=checkpoint" \
+  --jq '.artifacts[].id' \
+  | xargs -I{} gh api -X DELETE "repos/NVIDIA/aicr/actions/artifacts/{}"
+```
+
+The first v2 run then watches forward from the current head; historical entries
+predating the baseline are covered by release-time verification (the `aicr
+verify` path), not by this monitor.
+
 ## Reviewing Recipe Contributions
 
 A recipe PR touches `recipes/overlays/`, `recipes/mixins/`,

pkg/recipe/metadata.go

@@ -1315,19 +1315,28 @@ func (s *RecipeMetadataSpec) TopologicalLevels() ([][]string, error) {
 	return ComponentRefsTopologicalLevels(s.ComponentRefs)
 }
 
-// ComponentRefsTopologicalLevels is the free-function form of
-// RecipeMetadataSpec.TopologicalLevels — operates on a bare
-// []ComponentRef slice. Callers that have refs but not a full
-// RecipeMetadataSpec (e.g., the bundler post-resolution) use this.
-func ComponentRefsTopologicalLevels(refs []ComponentRef) ([][]string, error) {
+// buildDependencyGraph constructs the dependency graph shared by
+// TopologicalSort and ComponentRefsTopologicalLevels. It centralizes the
+// enabled-filtering and external-satisfaction semantics so the two traversals
+// (flat Kahn sort vs. level-grouped BFS) stay in lock-step — the duplication
+// this removes is exactly what caused the double-fix in #1465 (see #1466).
+//
+// Only enabled components are nodes. A dependency edge pointing at a declared-
+// but-disabled component is treated as already satisfied (the dependency is
+// assumed provided externally, e.g. a CSP-managed cert-manager) and excluded
+// from the in-degree count. An edge to an undeclared component is retained so
+// it still surfaces as a cycle/missing-dependency error. See componentSets and
+// edgeSatisfiedExternally.
+//
+// Returns the per-node in-degree, the reverse adjacency (dependency name → the
+// components that depend on it), and the number of enabled nodes. Callers
+// compare their processed count against enabledCount to detect cycles/missing
+// dependencies (a node whose in-degree never drains to zero is never emitted).
+func buildDependencyGraph(refs []ComponentRef) (inDegree map[string]int, dependents map[string][]string, enabledCount int) {
 	declared, enabled := componentSets(refs)
 
-	// Only enabled components are nodes; an edge pointing at a declared-but-
-	// disabled (externally-provided) component is treated as satisfied, while
-	// an edge to an undeclared component is retained so it still surfaces as a
-	// cycle/missing-dependency error. See componentSets and TopologicalSort.
-	inDegree := make(map[string]int, len(enabled))
-	dependents := make(map[string][]string, len(enabled))
+	inDegree = make(map[string]int, len(enabled))
+	dependents = make(map[string][]string, len(enabled))
 	for _, c := range refs {
 		if _, ok := enabled[c.Name]; !ok {
 			continue
@@ -1342,6 +1351,15 @@ func ComponentRefsTopologicalLevels(refs []ComponentRef) ([][]string, error) {
 		}
 		inDegree[c.Name] = degree
 	}
+	return inDegree, dependents, len(enabled)
+}
+
+// ComponentRefsTopologicalLevels is the free-function form of
+// RecipeMetadataSpec.TopologicalLevels — operates on a bare
+// []ComponentRef slice. Callers that have refs but not a full
+// RecipeMetadataSpec (e.g., the bundler post-resolution) use this.
+func ComponentRefsTopologicalLevels(refs []ComponentRef) ([][]string, error) {
+	inDegree, dependents, enabledCount := buildDependencyGraph(refs)
 
 	// Seed level 0: components with no incoming edges.
 	current := make([]string, 0, len(inDegree))
@@ -1371,7 +1389,7 @@ func ComponentRefsTopologicalLevels(refs []ComponentRef) ([][]string, error) {
 		current = next
 	}
 
-	if processed != len(enabled) {
+	if processed != enabledCount {
 		return nil, errors.New(errors.ErrCodeInvalidRequest,
 			"cannot determine deployment levels: circular dependencies exist")
 	}
@@ -1382,30 +1400,7 @@ func ComponentRefsTopologicalLevels(refs []ComponentRef) ([][]string, error) {
 // Components with no dependencies come first, then components that depend only
 // on already-listed components, etc.
 func (s *RecipeMetadataSpec) TopologicalSort() ([]string, error) {
-	declared, enabled := componentSets(s.ComponentRefs)
-
-	// Only enabled components are nodes. A dependency edge pointing at a
-	// declared-but-disabled component is treated as already satisfied (the
-	// dependency is assumed to be provided externally, e.g. a CSP-managed
-	// cert-manager) and excluded from the in-degree count. An edge to an
-	// undeclared component is still counted so it surfaces as a cycle/missing
-	// dependency error, matching the prior behavior. See componentSets.
-	inDegree := make(map[string]int, len(enabled))
-	dependents := make(map[string][]string) // dep -> list of components that depend on it
-	for _, c := range s.ComponentRefs {
-		if _, ok := enabled[c.Name]; !ok {
-			continue
-		}
-		degree := 0
-		for _, dep := range c.DependencyRefs {
-			if edgeSatisfiedExternally(dep, declared, enabled) {
-				continue
-			}
-			degree++
-			dependents[dep] = append(dependents[dep], c.Name)
-		}
-		inDegree[c.Name] = degree
-	}
+	inDegree, dependents, enabledCount := buildDependencyGraph(s.ComponentRefs)
 
 	// Kahn's algorithm
 	// https://www.geeksforgeeks.org/dsa/topological-sorting-indegree-based-solution/
@@ -1418,7 +1413,7 @@ func (s *RecipeMetadataSpec) TopologicalSort() ([]string, error) {
 	// Sort queue for deterministic output
 	sort.Strings(queue)
 
-	result := make([]string, 0, len(enabled))
+	result := make([]string, 0, enabledCount)
 	for len(queue) > 0 {
 		node := queue[0]
 		queue = queue[1:]
@@ -1434,7 +1429,7 @@ func (s *RecipeMetadataSpec) TopologicalSort() ([]string, error) {
 	}
 
 	// Check if all enabled nodes were processed (no cycles)
-	if len(result) != len(enabled) {
+	if len(result) != enabledCount {
 		return nil, errors.New(errors.ErrCodeInvalidRequest, "cannot determine deployment order: circular dependencies exist")
 	}