feat(recipe): add generation-time runtime inventory selection (#2317)

Signed-off-by: Mark Chmarny <mark@chmarny.com>

Author: mchmarny

api/aicr/v1/server.yaml

@@ -2622,6 +2622,13 @@ components:
                     mode:
                       type: string
                       enum: [disabled, customer-managed, aicr-provided]
+            runtimeInventory:
+              type: object
+              required: [mode]
+              properties:
+                mode:
+                  type: string
+                  enum: [enabled, disabled]
         componentRefs:
           type: array
           items:
@@ -2812,7 +2819,10 @@ components:
       allOf:
         - $ref: "#/components/schemas/RecipeResponseBase/properties/configuration"
         - type: object
-          required: [slurm]
+          # At least one section, but not any particular one: a recipe may
+          # record a runtime-inventory selection without Slurm accounting, or
+          # the reverse. Requiring `slurm` rejected the former outright.
+          minProperties: 1
           additionalProperties: false
           properties:
             slurm:
@@ -2828,6 +2838,14 @@ components:
                     mode:
                       type: string
                       enum: [disabled, customer-managed, aicr-provided]
+            runtimeInventory:
+              type: object
+              required: [mode]
+              additionalProperties: false
+              properties:
+                mode:
+                  type: string
+                  enum: [enabled, disabled]
 
     ProfileRecipeResponse:
       allOf:

docs/design/019-k8s-aibom-runtime-inventory.md

@@ -555,9 +555,9 @@ semantics.
 
 ## Follow-Up Decisions
 
-Three of the six requirements below are resolved by
+Four of the six requirements below are resolved by
 [Amendment: stock adoption on one GKE recipe](#amendment-stock-adoption-on-one-gke-recipe);
-one remains open, and two are planned work tracked in the epic. The list is
+the remaining two are planned work tracked in the epic. The list is
 retained as originally written; the amendment records what changed, what
 remains open, and where the rest is tracked.
 
@@ -634,25 +634,50 @@ cluster and establishing the pattern for later stock adoptions. This is
 recorded plainly rather than framed as customer demand, because the
 requirement exists to prevent adoption justified only by availability.
 
-### Still open: selection and opt-out semantics
+### E. Selection and opt-out semantics
 
-Unresolved and deliberately not decided here. `ComponentRef.IsEnabled()`
-already reads a recipe-recorded `enabled` override, which satisfies "recipe-
-recorded" and is not the bundle-time toggle Decision 2 rejects. What remains
-undecided is how a user generating from a stock recipe declines the
-component: by authoring a custom overlay using the existing mechanism, or by
-a generation-time flag that `aicr recipe` records into the emitted recipe.
+Resolved 2026-08-20. A **generation-time flag recorded in the emitted recipe**,
+modelled on the existing `--slurm-accounting-mode` selection rather than
+invented:
 
-The second shape changes the CLI contract, so it is a decision rather than an
-implementation detail. It must be resolved before the overlay change in C
-merges, and whichever shape is chosen is recorded by amending this section.
+```bash
+aicr recipe ... --runtime-inventory disabled
+```
+
+The selection is recorded as `configuration.runtimeInventory.mode`, the recipe's
+`apiVersion` becomes `ConfiguredRecipeResultAPIVersion`, and the component's ref
+carries `install: false`. `ComponentRef.IsEnabled()` already reads that key, so
+the component leaves the resolved set, the bundle, and deployment validation.
+
+This satisfies Decision 2's specific objection. A bundle-time
+`--set k8s-aibom:enabled=false` was rejected because it changes neither the
+recipe nor its health checks; here both change, and the health-check half comes
+for free because the check lives on the component's own ref rather than on a
+sibling. That is simpler than the Slurm accounting precedent, which has to
+append and omit a check on a different component.
+
+Passing the flag on a recipe that does not declare the component is an error,
+not a silent no-op. Selecting a mode there is a mistake — wrong criteria, a
+typo, a recipe that never carried it — and succeeding quietly would record a
+decision the recipe cannot honor. The check runs before the configuration is
+written, so a rejected build leaves no partial record.
+
+The same selection is available in an `AICRConfig` document at
+`spec.recipe.configuration.runtimeInventory.mode`.
+
+**Scope boundary worth naming.** This is the second entry under
+`RecipeConfiguration`, and the pattern is one bespoke selection per optional
+component. That is deliberate: this ADR asks for this component specifically,
+and a generic per-component disable would need a policy for which components
+may be declined at all — nothing should let a recipe decline `gpu-operator`.
+A third entry is the signal to revisit rather than extend by reflex.
 
 ### Requirement status
 
 | Follow-Up requirement | Status |
 |---|---|
 | Exact recipe families in scope | Resolved — C |
-| Selection and opt-out semantics | **Open** — see above |
+| Selection and opt-out semantics | Resolved — E |
 | Non-alpha storage API and migration policy | Resolved — A |
 | Concrete user-demand case | Resolved — D |
 | Managed-cluster qualification and measured cost | Planned — [#2271](https://github.com/NVIDIA/aicr/issues/2271) |

docs/integrator/go-library.md

@@ -721,7 +721,7 @@ func resolveCommittedConfig(ctx context.Context) (retErr error) {
 	if err != nil {
 		return err
 	}
-	opts, err := cfg.RecipeResolveOptions() // spec.recipe.profile + accounting mode
+	opts, err := cfg.RecipeResolveOptions() // profile + accounting + runtime inventory
 	if err != nil {
 		return err
 	}
@@ -769,8 +769,8 @@ derive step rather than the load step.
 | `BundleVerifyOptions()` | `spec.verify.policy` + `spec.verify.trust` |
 | `RecipeSource()` | `spec.recipe.data` |
 | `RecipeCriteria(reg)` | `spec.recipe.criteria` |
-| `RecipeResolveOptions()` | `spec.recipe.profile`, `spec.recipe.configuration.slurm.accounting.mode` |
-| `RecipeProfile()` / `RecipeAccountingMode()` | the same two, raw, for callers applying their own precedence first |
+| `RecipeResolveOptions()` | `spec.recipe.profile`, `spec.recipe.configuration.slurm.accounting.mode`, `spec.recipe.configuration.runtimeInventory.mode` |
+| `RecipeProfile()` / `RecipeAccountingMode()` / `RecipeRuntimeInventoryMode()` | the same three, raw, for callers applying their own precedence first |
 | `SnapshotPath()` | `spec.recipe.input.snapshot` |
 | `IsCriteriaStrict()` | `spec.recipe.criteriaStrict` |
 

docs/user/cli-reference.md

@@ -459,6 +459,7 @@ Generate recipes using direct system parameters:
 | `--platform` | | string | Platform/framework type: dynamo, kubeflow, nim, runai, slurm |
 | `--profile` | | string | Profile selection in exact `name=value` form (e.g. `gpuStack=operator-managed` on AKS or `gpuStack=driver-installer` on GKE); omit to use the declaration's default (`gpuStack=azure-managed` on AKS, `gpuStack=gke-default` on GKE) |
 | `--slurm-accounting-mode` | | string | Slurm accounting ownership: disabled (default), customer-managed, aicr-provided |
+| `--runtime-inventory` | | string | Runtime AI inventory (`k8s-aibom`) selection: `enabled`, `disabled`. Recorded in the generated recipe |
 | `--nodes` | | int | Number of GPU nodes in the cluster |
 | `--output` | `-o` | string | Output file (default: stdout) |
 | `--format` | `-t` | string | Format: json, yaml, table (default: yaml) |
@@ -534,6 +535,7 @@ target-cluster conflict detection.
 | `--platform` | | string | Explicit platform/framework type, including slurm |
 | `--profile` | | string | Profile selection in exact `name=value` form; omit to use the declaration's default |
 | `--slurm-accounting-mode` | | string | Slurm accounting ownership: disabled (default), customer-managed, aicr-provided |
+| `--runtime-inventory` | | string | Runtime AI inventory (`k8s-aibom`) selection: `enabled`, `disabled`. Recorded in the generated recipe |
 | `--output` | `-o` | string | Output destination (file, ConfigMap URI, or stdout) |
 | `--format` | `-t` | string | Format: json, yaml, table (default: yaml) |
 | `--kubeconfig` | `-k` | string | Path to kubeconfig file (used when `--snapshot` or `--output` is a ConfigMap URI; overrides KUBECONFIG env) |

docs/user/component-catalog.md

@@ -366,6 +366,41 @@ storage. So the check catches a stranded upgrade that crosses a
 storage-version boundary, such as the 1.2.0 to 1.3.0 move this pin made, and
 does not catch one within a boundary, such as 1.0.0 to 1.2.0.
 
+**Declining the component.** No stock recipe declares `k8s-aibom` today, so the
+flag applies to a recipe that adds it through a custom overlay — the shape shown
+above. Point `--data` at the directory holding that overlay and generate with
+`--runtime-inventory disabled`:
+
+```bash
+aicr recipe --service gke --accelerator h100 --os cos --intent inference \
+  --data ./my-recipes --runtime-inventory disabled -o recipe.yaml
+```
+
+Passing the flag against a recipe that does not declare the component is an
+error, not a silent no-op:
+
+```console
+$ aicr recipe --service gke --accelerator h100 --os cos --intent inference \
+    --runtime-inventory disabled
+[INVALID_REQUEST] runtime inventory mode "disabled" requires the recipe to
+declare component "k8s-aibom"; this recipe does not resolve it
+```
+
+The selection is recorded in the emitted recipe as
+`configuration.runtimeInventory.mode`, and the component's ref carries
+`install: false`, so the component and its health check are both absent from
+the bundle and from deployment validation. A bundle-time
+`--set k8s-aibom:enabled=false` is **not** equivalent and is not a supported
+way to decline the component: it changes neither the recipe nor its health
+checks, which is why [ADR-019](https://github.com/NVIDIA/aicr/blob/main/docs/design/019-k8s-aibom-runtime-inventory.md)
+rejects it as a selection contract.
+
+A wrong `--service` or a typo therefore surfaces instead of producing a recipe
+that claims a decision it never applied.
+
+The same selection is available in an `AICRConfig` document as
+`spec.recipe.configuration.runtimeInventory.mode`.
+
 **Overriding the chart version requires overriding this assertion.** Assert
 content is static YAML with no templating, so the expected storage version is
 a literal tied to the registry's pinned chart, currently `v1beta1` for chart

pkg/cli/consts.go

@@ -37,6 +37,7 @@ const (
 	flagPlatform            = "platform"
 	flagProfile             = "profile"
 	flagSlurmAccountingMode = "slurm-accounting-mode"
+	flagRuntimeInventory    = "runtime-inventory"
 	flagNoHealth            = "no-health"
 )
 

pkg/cli/query.go

@@ -96,7 +96,7 @@ Use in shell scripts:
 		Flags: queryCmdFlags(),
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			if err := validateSingleValueFlags(cmd, "service", "accelerator", "intent", "os", "platform",
-				flagProfile, flagSlurmAccountingMode, "snapshot", "config", "format", "selector"); err != nil {
+				flagProfile, flagSlurmAccountingMode, flagRuntimeInventory, "snapshot", "config", "format", "selector"); err != nil {
 				return err
 			}
 
@@ -164,7 +164,7 @@ Use in shell scripts:
 func buildRecipeFromCmdWithConfig(ctx context.Context, cmd *cli.Command, cfg *appcfg.AICRConfig, client *aicr.Client) (*aicr.RecipeResult, error) {
 	reg := client.CriteriaRegistry()
 	profile := stringFlagOrConfig(cmd, flagProfile, aicr.WrapConfig(cfg).RecipeProfile())
-	resolveOpts, err := accountingResolveOptions(cmd, cfg)
+	resolveOpts, err := buildSelectionResolveOptions(cmd, cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -263,6 +263,45 @@ func accountingResolveOptions(cmd *cli.Command, cfg *appcfg.AICRConfig) ([]aicr.
 	return []aicr.RecipeResolveOption{aicr.WithAccountingMode(value)}, nil
 }
 
+// runtimeInventoryResolveOptions turns the --runtime-inventory flag, or the
+// equivalent AICRConfig field, into a resolve option. Mirrors
+// accountingResolveOptions: the flag wins, the config file is the fallback,
+// and an absent selection leaves the recipe's own declaration alone.
+func runtimeInventoryResolveOptions(cmd *cli.Command, cfg *appcfg.AICRConfig) ([]aicr.RecipeResolveOption, error) {
+	value := cmd.String(flagRuntimeInventory)
+	if !cmd.IsSet(flagRuntimeInventory) {
+		if cfg == nil {
+			return nil, nil
+		}
+		mode, present, err := aicr.WrapConfig(cfg).RecipeRuntimeInventoryMode()
+		if err != nil {
+			return nil, err
+		}
+		if !present {
+			return nil, nil
+		}
+		value = mode
+	}
+	if _, err := recipe.ParseRuntimeInventoryMode(value); err != nil {
+		return nil, err
+	}
+	return []aicr.RecipeResolveOption{aicr.WithRuntimeInventoryMode(value)}, nil
+}
+
+// buildSelectionResolveOptions gathers every generation-time selection into one
+// option slice, so callers cannot wire one and forget the other.
+func buildSelectionResolveOptions(cmd *cli.Command, cfg *appcfg.AICRConfig) ([]aicr.RecipeResolveOption, error) {
+	opts, err := accountingResolveOptions(cmd, cfg)
+	if err != nil {
+		return nil, err
+	}
+	riOpts, err := runtimeInventoryResolveOptions(cmd, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return append(opts, riOpts...), nil
+}
+
 // statedDimensions converts the touched set into the argument
 // aicr.WithSnapshotCriteriaRelaxation expects: the dimensions the user stated
 // explicitly, which the facade must never relax. Order is canonical so the

pkg/cli/query_test.go

@@ -291,3 +291,70 @@ spec:
 	}
 	return dir
 }
+
+// TestRecipeAndQueryCommandsRejectInvalidRuntimeInventoryMode exercises
+// runtimeInventoryResolveOptions' flag-set branch on both commands that expose
+// the flag.
+//
+// Without this the function is covered only by its "flag unset and no config"
+// fallthrough, which returns before touching the value — so a parse or wiring
+// defect in the branch operators actually use would not be observed.
+func TestRecipeAndQueryCommandsRejectInvalidRuntimeInventoryMode(t *testing.T) {
+	tests := []struct {
+		name string
+		cmd  func() *cli.Command
+		args []string
+	}{
+		{
+			name: "recipe",
+			cmd:  recipeCmd,
+			args: []string{"recipe", "--service", "eks", "--runtime-inventory", "off"},
+		},
+		{
+			name: "query",
+			cmd:  queryCmd,
+			args: []string{
+				"query", "--service", "eks", "--selector", "deploymentOrder",
+				"--runtime-inventory", "off",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cmd().Run(t.Context(), tt.args)
+			if err == nil {
+				t.Fatal("command error = nil, want rejection of an invalid runtime inventory mode")
+			}
+			// Assert the code, since that is what callers branch on; a text
+			// match alone would accept an unrelated error carrying similar
+			// wording. The message check stays to distinguish which
+			// invalid-request this is.
+			if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) {
+				t.Errorf("command error = %v, want ErrCodeInvalidRequest", err)
+			}
+			if !strings.Contains(err.Error(), "invalid runtime inventory mode") {
+				t.Fatalf("command error = %v, want an invalid-mode rejection", err)
+			}
+		})
+	}
+}
+
+// TestRecipeCommandRejectsRuntimeInventoryWithoutComponent covers the flag-set
+// branch reaching a successful parse and then failing closed at build time,
+// which is the path an operator hits after a typo in --service.
+func TestRecipeCommandRejectsRuntimeInventoryWithoutComponent(t *testing.T) {
+	err := recipeCmd().Run(t.Context(), []string{
+		"recipe", "--service", "gke", "--accelerator", "h100",
+		"--os", "cos", "--intent", "inference",
+		"--runtime-inventory", "disabled",
+	})
+	if err == nil {
+		t.Fatal("command error = nil, want rejection for a recipe that does not declare the component")
+	}
+	if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) {
+		t.Errorf("command error = %v, want ErrCodeInvalidRequest", err)
+	}
+	if !strings.Contains(err.Error(), "requires the recipe to declare component") {
+		t.Fatalf("command error = %v, want the missing-component rejection", err)
+	}
+}

pkg/cli/recipe.go

@@ -76,6 +76,14 @@ func recipeCmdFlags() []cli.Flag {
 			Usage:    fmt.Sprintf("Slurm accounting ownership mode (%s)", strings.Join(recipe.AccountingModes(), ", ")),
 			Category: catQueryParameters,
 		}, recipe.AccountingModes),
+		withCompletions(&cli.StringFlag{
+			Name: flagRuntimeInventory,
+			Usage: fmt.Sprintf(
+				"Runtime AI inventory (k8s-aibom) selection (%s). Recorded in the generated recipe; "+
+					"disabling removes the component and its health check",
+				strings.Join(recipe.RuntimeInventoryModes(), ", ")),
+			Category: catQueryParameters,
+		}, recipe.RuntimeInventoryModes),
 		&cli.IntFlag{
 			Name:     "nodes",
 			Usage:    "Number of worker/GPU nodes in the cluster",
@@ -149,7 +157,7 @@ Override snapshot-detected criteria:
 		Flags: recipeCmdFlags(),
 		Action: func(ctx context.Context, cmd *cli.Command) error {
 			if err := validateSingleValueFlags(cmd, flagService, flagAccelerator, flagIntent, flagOS,
-				flagPlatform, flagProfile, flagSlurmAccountingMode, "snapshot", "config", flagOutput,
+				flagPlatform, flagProfile, flagSlurmAccountingMode, flagRuntimeInventory, "snapshot", "config", flagOutput,
 				flagFormat); err != nil {
 				return err
 			}

pkg/client/v1/aicr.go

@@ -859,10 +859,14 @@ func recipeBuildOptions(opts ...RecipeResolveOption) (*recipeResolveConfig, []re
 	if err != nil {
 		return nil, nil, err
 	}
-	if cfg.accountingMode == nil {
-		return cfg, nil, nil
+	var buildOpts []recipe.BuildOption
+	if cfg.accountingMode != nil {
+		buildOpts = append(buildOpts, recipe.WithAccountingMode(*cfg.accountingMode))
 	}
-	return cfg, []recipe.BuildOption{recipe.WithAccountingMode(*cfg.accountingMode)}, nil
+	if cfg.runtimeInventoryMode != nil {
+		buildOpts = append(buildOpts, recipe.WithRuntimeInventoryMode(*cfg.runtimeInventoryMode))
+	}
+	return cfg, buildOpts, nil
 }
 
 func resolveRecipeConfig(opts ...RecipeResolveOption) (*recipeResolveConfig, error) {