OCPBUGS-84150: MachineOSConfig "build was not found" error

pkg/controller/build/reconciler.go

@@ -1347,18 +1347,29 @@ func (b *buildReconciler) reconcilePoolChange(ctx context.Context, mcp *mcfgv1.M
 	// old pool
 	old := mcp.DeepCopy()
 	old.Spec.Configuration.Name = mcp.Status.Configuration.Name
-	if firstOptIn == "" {
-		return fmt.Errorf("no current build annotation on MachineOSConfig %q", mosc.Name)
-	}
 
+	// Always determine if this change requires an image rebuild or is layer-only
 	needsImageRebuild, err := b.reconcileImageRebuild(old, mcp)
 	if err != nil {
 		return err
 	}
 
+	// If annotation is empty, this is either:
+	// 1. First time opting into OCL, or
+	// 2. Node controller cleared a stale annotation
+	// In both cases, we need to create/continue a build
+	missingAnnotation := firstOptIn == ""
+
 	// This is our trigger point
-	if (oldRendered != newRendered && needsImageRebuild) || firstOptIn == "" {
-		klog.Infof("pool %q: rendered config changed and requires an image rebuild. Verifying if a valid build already exists...", mcp.Name)
+	// Create/continue a build if:
+	// 1. Config changed AND needs image rebuild, OR
+	// 2. Annotation is missing (first opt-in or cleared due to staleness)
+	if (oldRendered != newRendered && needsImageRebuild) || missingAnnotation {
+		if needsImageRebuild {
+			klog.Infof("pool %q: rendered config changed and requires an image rebuild. Verifying if a valid build already exists...", mcp.Name)
+		} else {
+			klog.Infof("pool %q: rendered config changed (layer-only) or annotation missing. Verifying if a valid build already exists...", mcp.Name)
+		}
 		mc, err := b.machineConfigLister.Get(mcp.Spec.Configuration.Name)
 		if err != nil {
 			return fmt.Errorf("could not get MachineConfig %q for MachineConfigPool %q: %w", mcp.Spec.Configuration.Name, mcp.Name, err)

pkg/controller/node/node_controller.go

@@ -1175,6 +1175,7 @@ func (ctrl *Controller) getConfigAndBuild(pool *mcfgv1.MachineConfigPool) (*mcfg
 
 	// First, try to get the MOSB from the current-machine-os-build annotation on the MOSC
 	// This ensures we get the correct build when multiple MOSBs exist for the same rendered MC
+	staleAnnotation := false
 	if currentBuildName, hasAnnotation := ourConfig.Annotations[buildconstants.CurrentMachineOSBuildAnnotationKey]; hasAnnotation {
 		for _, build := range buildList {
 			if build.Name == currentBuildName {
@@ -1186,11 +1187,32 @@ func (ctrl *Controller) getConfigAndBuild(pool *mcfgv1.MachineConfigPool) (*mcfg
 					return ourConfig, ourBuild, nil
 				}
 				klog.Warningf("MachineOSBuild %q from annotation is for rendered config %q, but pool has %q - annotation is stale", currentBuildName, build.Spec.MachineConfig.Name, pool.Spec.Configuration.Name)
-				// Don't return here - fall through to the fallback logic
+				staleAnnotation = true
 				break
 			}
 		}
-		klog.Warningf("MachineOSConfig %q has current-machine-os-build annotation pointing to %q, but that build was not found", ourConfig.Name, currentBuildName)
+		if !staleAnnotation {
+			// Build not found in lister - verify with live API call to avoid cache race conditions
+			// where MOSC informer cache updates before MOSB informer cache
+			_, err := ctrl.client.MachineconfigurationV1().MachineOSBuilds().Get(context.TODO(), currentBuildName, metav1.GetOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				return nil, nil, fmt.Errorf("failed to verify MachineOSBuild %q exists: %w", currentBuildName, err)
+			}
+			if errors.IsNotFound(err) {
+				klog.Warningf("MachineOSConfig %q has current-machine-os-build annotation pointing to %q, but that build was not found", ourConfig.Name, currentBuildName)
+				staleAnnotation = true
+			}
+		}
+
+		// Clear stale annotation to allow build controller to create a new build
+		if staleAnnotation {
+			klog.Infof("Clearing stale current-machine-os-build annotation %q from MachineOSConfig %q", currentBuildName, ourConfig.Name)
+			configCopy := ourConfig.DeepCopy()
+			delete(configCopy.Annotations, buildconstants.CurrentMachineOSBuildAnnotationKey)
+			if _, err := ctrl.client.MachineconfigurationV1().MachineOSConfigs().Update(context.TODO(), configCopy, metav1.UpdateOptions{}); err != nil {
+				return nil, nil, fmt.Errorf("failed to clear stale annotation from MachineOSConfig %q: %w", ourConfig.Name, err)
+			}
+		}
 	}
 
 	// Fallback: if annotation is not present or build not found, use the old logic