Skip to content

Commit 6405b3e

Browse files
Copilotpelikhan
andauthored
Fix Daily Formal Spec Verifier safe output contract
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
1 parent 94079de commit 6405b3e

3 files changed

Lines changed: 38 additions & 2 deletions

File tree

.github/workflows/daily-formal-spec-verifier.lock.yml

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

.github/workflows/daily-formal-spec-verifier.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,13 @@ Use existing types, functions, and interfaces from the codebase where possible (
174174

175175
Create exactly one issue using the `create_issue` safe output.
176176

177+
### Output Contract (Required)
178+
179+
1. Draft the title and body locally first if needed, but emit exactly one final `create_issue` safe output only after the full payload is complete.
180+
2. Do **not** use `bash`, `cli-proxy`, or the `safeoutputs` CLI to create the issue or inspect the tool schema. Emit the safe output directly with `title` and `body` arguments.
181+
3. Never retry `create_issue` with empty, placeholder, or partial arguments.
182+
4. If the quality checks below cannot be met, emit `report_incomplete` directly as a safe output instead of `create_issue`.
183+
177184
### Issue format
178185

179186
Title: `[formal-spec] <SpecFileName> — Formal model & test suite — <YYYY-MM-DD>`
@@ -239,7 +246,7 @@ Before emitting `create_issue`, verify the body:
239246
- The generated test file compiles without errors (review for syntax mistakes).
240247
- Is at least 1200 characters long.
241248

242-
If these checks cannot be met, emit `report_incomplete` instead of `create_issue`.
249+
If these checks cannot be met, emit `report_incomplete` directly as a safe output instead of `create_issue`.
243250

244251
---
245252

pkg/workflow/prompts_test.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,35 @@ func TestDailyCavemanOptimizerUsesConcreteClaudeModelsForExperiment(t *testing.T
324324
}
325325
}
326326

327+
func TestDailyFormalSpecVerifierDefinesDirectSafeOutputContract(t *testing.T) {
328+
repoRoot, err := findRepoRoot()
329+
if err != nil {
330+
t.Fatalf("Failed to find repo root: %v", err)
331+
}
332+
333+
workflowFile := filepath.Join(repoRoot, ".github", "workflows", "daily-formal-spec-verifier.md")
334+
content, err := os.ReadFile(workflowFile)
335+
if err != nil {
336+
t.Fatalf("Failed to read workflow file: %v", err)
337+
}
338+
339+
workflow := string(content)
340+
requiredContract := "Draft the title and body locally first if needed, but emit exactly one final `create_issue` safe output only after the full payload is complete."
341+
if !strings.Contains(workflow, requiredContract) {
342+
t.Fatal("Expected daily-formal-spec-verifier workflow to require a single final create_issue safe output")
343+
}
344+
345+
noShellGuidance := "Do **not** use `bash`, `cli-proxy`, or the `safeoutputs` CLI to create the issue or inspect the tool schema. Emit the safe output directly with `title` and `body` arguments."
346+
if !strings.Contains(workflow, noShellGuidance) {
347+
t.Fatal("Expected daily-formal-spec-verifier workflow to forbid bash/CLI safe-output invocation")
348+
}
349+
350+
reportIncompleteGuidance := "If the quality checks below cannot be met, emit `report_incomplete` directly as a safe output instead of `create_issue`."
351+
if !strings.Contains(workflow, reportIncompleteGuidance) {
352+
t.Fatal("Expected daily-formal-spec-verifier workflow to require direct report_incomplete fallback")
353+
}
354+
}
355+
327356
func TestDailyCacheStrategyAnalyzerUsesCodexCompatibleModelsForExperiment(t *testing.T) {
328357
repoRoot, err := findRepoRoot()
329358
if err != nil {

0 commit comments

Comments
 (0)