Skip to content

Commit b305be7

Browse files
ShmuelOpsclaude
andcommitted
fix: re-sync PrefectAutomation every reconcile for continuous self-heal
PrefectAutomation persisted Status.LastSyncTime, so needsSync short-circuited on a 10-minute drift timer and never re-checked Prefect in between. A manual edit or delete of the automation in Prefect was therefore not reconciled until the drift window elapsed or the operator restarted. PrefectDeployment reconciles against Prefect on every pass (its status carries no LastSyncTime), so it self-heals continuously. Align PrefectAutomation with that: needsSync now returns true each reconcile. syncWithPrefect is idempotent (update-in-place; with #297, recreate-on-404), so re-running it every pass is safe. Adds a unit test asserting needsSync returns true in steady state. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent fa792dc commit b305be7

2 files changed

Lines changed: 76 additions & 4 deletions

File tree

internal/controller/prefectautomation_controller.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,14 @@ func (r *PrefectAutomationReconciler) needsSync(automation *prefectiov1.PrefectA
120120
if automation.Status.ObservedGeneration < automation.Generation {
121121
return true
122122
}
123-
if automation.Status.LastSyncTime == nil {
124-
return true
125-
}
126-
return time.Since(automation.Status.LastSyncTime.Time) > 10*time.Minute
123+
// Otherwise re-sync on every reconcile, rather than gating on a coarse drift
124+
// timer. PrefectDeployment reconciles against Prefect each loop (its status
125+
// carries no LastSyncTime) and so self-heals out-of-band changes continuously;
126+
// PrefectAutomation persisted LastSyncTime and so only re-checked every 10m,
127+
// meaning a manual edit/delete in Prefect wasn't reconciled until the drift
128+
// window elapsed or the operator restarted. syncWithPrefect is idempotent, so
129+
// re-running it each pass is safe.
130+
return true
127131
}
128132

129133
// syncWithPrefect creates or updates the automation in the Prefect API
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller
18+
19+
import (
20+
"testing"
21+
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
24+
prefectiov1 "github.com/PrefectHQ/prefect-operator/api/v1"
25+
)
26+
27+
// needsSync must return true on every reconcile so the controller re-checks the
28+
// automation against Prefect each loop and self-heals an out-of-band edit/delete
29+
// (matching PrefectDeployment's continuous reconcile) rather than only on a
30+
// coarse drift timer or an operator restart.
31+
func TestAutomationNeedsSync(t *testing.T) {
32+
r := &PrefectAutomationReconciler{}
33+
id := "automation-1"
34+
now := metav1.Now()
35+
36+
newHash := "hash-1"
37+
base := func() *prefectiov1.PrefectAutomation {
38+
a := &prefectiov1.PrefectAutomation{}
39+
a.Generation = 2
40+
a.Status.Id = &id
41+
a.Status.SpecHash = newHash
42+
a.Status.ObservedGeneration = 2
43+
a.Status.LastSyncTime = &now
44+
return a
45+
}
46+
47+
cases := []struct {
48+
name string
49+
mutate func(*prefectiov1.PrefectAutomation)
50+
hash string
51+
wantMsg string
52+
}{
53+
{"no id", func(a *prefectiov1.PrefectAutomation) { a.Status.Id = nil }, newHash, "unsynced automation"},
54+
{"spec changed", func(a *prefectiov1.PrefectAutomation) {}, "different-hash", "spec change"},
55+
{"generation ahead", func(a *prefectiov1.PrefectAutomation) { a.Generation = 3 }, newHash, "generation bump"},
56+
{"steady state (just synced)", func(a *prefectiov1.PrefectAutomation) {}, newHash, "continuous self-heal"},
57+
}
58+
59+
for _, tc := range cases {
60+
t.Run(tc.name, func(t *testing.T) {
61+
a := base()
62+
tc.mutate(a)
63+
if !r.needsSync(a, tc.hash) {
64+
t.Fatalf("needsSync = false, want true (%s)", tc.wantMsg)
65+
}
66+
})
67+
}
68+
}

0 commit comments

Comments
 (0)