chore(rest-api): Align Site Agent and Site worker on inventory interval

devspace.yaml

@@ -303,9 +303,9 @@ deployments:
           CORE_GRPC_SEC_OPT: "2"
           FLOW_GRPC_ENABLED: "false"
           CLUSTER_ID: 00000000-0000-4000-8000-000000000001
-          # The REST workflow ignores inventory for 3m5s after a machine update.
-          # Keep the local inventory cadence beyond that guard to avoid skipped hosts.
-          TEMPORAL_INVENTORY_SCHEDULE: "@every 4m"
+          # The Site Agent reports this interval and Cloud derives its staleness guard from it,
+          # so a fast local cadence no longer needs padding to avoid skipped hosts.
+          TEMPORAL_INVENTORY_SCHEDULE: "@every 1m"
           TEMPORAL_SERVER: site.server.temporal.local
           TEMPORAL_SUBSCRIBE_NAMESPACE: 00000000-0000-4000-8000-000000000001
           TEMPORAL_SUBSCRIBE_QUEUE: site

rest-api/common/pkg/util/workflow.go

@@ -6,8 +6,14 @@ package util
 import "time"
 
 const (
-	// InventoryReceiptInterval is the interval between 2 subsequent inventory receipts
-	InventoryReceiptInterval = 3 * time.Minute
+	// DefaultInventoryReceiptInterval is the assumed interval between 2 subsequent inventory
+	// receipts for a Site that has not reported its own collection interval. Prefer
+	// Site.IsTimeWithinStaleInventoryThreshold, which follows the reported interval where there
+	// is one.
+	DefaultInventoryReceiptInterval = 3 * time.Minute
+	// StaleInventoryBuffer keeps the staleness check from sitting exactly on the collection
+	// interval, where clock skew between the Site and Cloud decides the outcome.
+	StaleInventoryBuffer = 10 * time.Second
 	// WorkflowExecutionTimeout is the timeout for a workflow execution
 	WorkflowExecutionTimeout = time.Minute * 1
 	// WorkflowContextTimeout is the timeout for a workflow context

rest-api/db/pkg/db/model/site.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"time"
 
+	cutil "github.com/NVIDIA/infra-controller/rest-api/common/pkg/util"
 	"github.com/NVIDIA/infra-controller/rest-api/db/pkg/db"
 	"github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/paginator"
 	stracer "github.com/NVIDIA/infra-controller/rest-api/db/pkg/tracer"
@@ -78,6 +79,7 @@ type Site struct {
 	SerialConsoleMaxSessionLength *int                    `bun:"serial_console_max_session_length"`
 	IsInfinityEnabled             bool                    `bun:"is_infinity_enabled,notnull"`
 	InventoryReceived             *time.Time              `bun:"inventory_received"`
+	InventoryIntervalSeconds      *int                    `bun:"inventory_interval_seconds"`
 	Status                        string                  `bun:"status,notnull"`
 	Created                       time.Time               `bun:"created,nullzero,notnull,default:current_timestamp"`
 	Updated                       time.Time               `bun:"updated,nullzero,notnull,default:current_timestamp"`
@@ -99,6 +101,19 @@ type SiteContact struct {
 	Email string `json:"email"`
 }
 
+// IsTimeWithinStaleInventoryThreshold reports whether actionTime is recent enough that an
+// arriving inventory may predate it, which means the inventory should not be acted on for that
+// object. The threshold follows the collection interval the Site Agent reports, and falls back
+// to the default for a Site that has not reported one yet, so an unreported Site keeps the
+// protection it had before the field existed.
+func (st *Site) IsTimeWithinStaleInventoryThreshold(actionTime time.Time) bool {
+	interval := cutil.DefaultInventoryReceiptInterval
+	if st != nil && st.InventoryIntervalSeconds != nil && *st.InventoryIntervalSeconds > 0 {
+		interval = time.Duration(*st.InventoryIntervalSeconds) * time.Second
+	}
+	return time.Since(actionTime) < interval+cutil.StaleInventoryBuffer
+}
+
 type SiteCreateInput struct {
 	Name                          string
 	DisplayName                   *string
@@ -148,6 +163,7 @@ type SiteUpdateInput struct {
 	SerialConsoleMaxSessionLength *int
 	IsInfinityEnabled             *bool
 	InventoryReceived             *time.Time
+	InventoryIntervalSeconds      *int
 	Status                        *string
 	Location                      *SiteLocation
 	Contact                       *SiteContact
@@ -545,6 +561,12 @@ func (ssd SiteSQLDAO) Update(ctx context.Context, tx *db.Tx, input SiteUpdateInp
 		ssd.tracerSpan.SetAttribute(stDAOSpan, "inventory_received", *input.InventoryReceived)
 	}
 
+	if input.InventoryIntervalSeconds != nil {
+		st.InventoryIntervalSeconds = input.InventoryIntervalSeconds
+		updatedFields = append(updatedFields, "inventory_interval_seconds")
+		ssd.tracerSpan.SetAttribute(stDAOSpan, "inventory_interval_seconds", *input.InventoryIntervalSeconds)
+	}
+
 	if input.Status != nil {
 		st.Status = *input.Status
 		updatedFields = append(updatedFields, "status")

rest-api/db/pkg/db/model/site_test.go

@@ -1893,3 +1893,82 @@ func validateSite(t *testing.T, got Site, want Site) {
 	assert.Equal(t, want.Location, got.Location)
 	assert.Equal(t, want.Contact, got.Contact)
 }
+
+func TestSite_IsTimeWithinStaleInventoryThreshold(t *testing.T) {
+	reportedOneMinute := &Site{InventoryIntervalSeconds: cutil.GetPtr(60)}
+
+	tests := []struct {
+		name       string
+		site       *Site
+		actionTime time.Time
+		want       bool
+	}{
+		{
+			name:       "a change just now is too recent to act on",
+			site:       &Site{},
+			actionTime: time.Now(),
+			want:       true,
+		},
+		{
+			name:       "a change older than the fallback threshold is safe to act on",
+			site:       &Site{},
+			actionTime: time.Now().Add(-(cutil.DefaultInventoryReceiptInterval + cutil.StaleInventoryBuffer + time.Second)),
+			want:       false,
+		},
+		{
+			// The same age that clears the fallback is still too recent for a slower Site.
+			name:       "follows a reported interval longer than the fallback",
+			site:       &Site{InventoryIntervalSeconds: cutil.GetPtr(600)},
+			actionTime: time.Now().Add(-(cutil.DefaultInventoryReceiptInterval + cutil.StaleInventoryBuffer + time.Second)),
+			want:       true,
+		},
+		{
+			// The same age is stale against the fallback and safe against a faster Site, which
+			// is the whole point of following the reported interval.
+			name:       "an age between the two intervals depends on the reported one",
+			site:       reportedOneMinute,
+			actionTime: time.Now().Add(-2 * time.Minute),
+			want:       false,
+		},
+		{
+			name:       "the same age is still too recent against the fallback",
+			site:       &Site{},
+			actionTime: time.Now().Add(-2 * time.Minute),
+			want:       true,
+		},
+		{
+			// The buffer keeps the check off the exact interval, where clock skew between the
+			// Site and Cloud would decide the outcome.
+			name:       "a change inside the buffer past the interval is still too recent",
+			site:       reportedOneMinute,
+			actionTime: time.Now().Add(-(time.Minute + cutil.StaleInventoryBuffer/2)),
+			want:       true,
+		},
+		{
+			// A stored zero or negative would otherwise collapse the threshold to the buffer
+			// alone and let inventory act on data it should treat as newer.
+			name:       "falls back on a zero reported interval",
+			site:       &Site{InventoryIntervalSeconds: cutil.GetPtr(0)},
+			actionTime: time.Now().Add(-(cutil.StaleInventoryBuffer + time.Second)),
+			want:       true,
+		},
+		{
+			name:       "falls back on a negative reported interval",
+			site:       &Site{InventoryIntervalSeconds: cutil.GetPtr(-30)},
+			actionTime: time.Now().Add(-(cutil.StaleInventoryBuffer + time.Second)),
+			want:       true,
+		},
+		{
+			name:       "falls back on a nil Site",
+			site:       nil,
+			actionTime: time.Now().Add(-(cutil.StaleInventoryBuffer + time.Second)),
+			want:       true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, tt.site.IsTimeWithinStaleInventoryThreshold(tt.actionTime))
+		})
+	}
+}

rest-api/db/pkg/migrations/20260821191500_site_inventory_interval.go

@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package migrations
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model"
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+		tx, terr := db.BeginTx(ctx, &sql.TxOptions{})
+		if terr != nil {
+			handlePanic(terr, "failed to begin transaction")
+		}
+
+		// Nullable because the Site Agent reports this on its Site Config inventory, so a Site
+		// stays NULL until its first report. Readers fall back to the built-in interval rather
+		// than treating a missing value as zero.
+		_, err := tx.NewAddColumn().
+			Model((*model.Site)(nil)).
+			IfNotExists().
+			ColumnExpr("inventory_interval_seconds INTEGER").
+			Exec(ctx)
+		handleError(tx, err)
+
+		terr = tx.Commit()
+		if terr != nil {
+			handlePanic(terr, "failed to commit transaction")
+		}
+
+		fmt.Print(" [up migration] Added 'inventory_interval_seconds' column to 'site' table successfully. ")
+		return nil
+	}, func(ctx context.Context, db *bun.DB) error {
+		// Safe to drop. The Site Agent re-reports the interval on its next inventory tick, so
+		// nothing here has to be reconstructed by hand.
+		_, err := db.ExecContext(ctx, `ALTER TABLE site DROP COLUMN IF EXISTS inventory_interval_seconds`)
+		if err != nil {
+			return err
+		}
+		fmt.Print(" [down migration] Dropped 'inventory_interval_seconds' column from 'site' table successfully. ")
+		return nil
+	})
+}