ai-engineering-from-scratch/phases/14-agent-engineering/38-verification-gates/quiz.json at main · zhaozhiqiangjianming-ship-it/ai-engineering-from-scratch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
{
  "lesson": "38-verification-gates",
  "title": "Verification Gates",
  "questions": [
    {
      "stage": "pre",
      "question": "What single question does the verification gate answer?",
      "options": [
        "Is the model fast?",
        "Is this task actually complete? (reading scope, rule, feedback, and diff artifacts)",
        "Is the prompt optimal?",
        "Is the token budget healthy?"
      ],
      "correct": 1,
      "explanation": "The gate is a deterministic function over workbench artifacts producing a pass/fail verdict."
    },
    {
      "stage": "pre",
      "question": "Why must the gate be deterministic?",
      "options": [
        "Determinism is free",
        "The same artifact set must produce the same verdict every time; LLM judges belong in the reviewer (qualitative), not the gate (status)",
        "Providers require it",
        "It saves money"
      ],
      "correct": 1,
      "explanation": "Mixing model judgment into the gate collapses the deterministic/qualitative split."
    },
    {
      "stage": "check",
      "question": "What is the gate's override discipline?",
      "options": [
        "Anyone can override silently",
        "Block-severity findings can only be overridden by a human with a recorded override_reason and overridden_by user id in a signed audit log",
        "Override requires a manager email",
        "Overrides are forbidden"
      ],
      "correct": 1,
      "explanation": "Signed overrides land in outputs/verification/overrides.jsonl; agent cannot self-override."
    },
    {
      "stage": "check",
      "question": "What is the Hybrid Norm pairing the lesson cites?",
      "options": [
        "Hot/cold prompts",
        "Verifiable rewards (tests, schemas, exit codes) answer 'did it solve the problem?'; LLM rubrics answer 'is it readable, secure, on-style?'",
        "GPU and CPU split",
        "Cache vs no-cache"
      ],
      "correct": 1,
      "explanation": "Anthropic 2026 guidance: gate runs the first class; reviewer (Lesson 39) runs the second."
    },
    {
      "stage": "check",
      "question": "How does defense-in-depth layer the gates?",
      "options": [
        "Single gate at merge time",
        "Pre-commit hook -> CI status check -> pre-tool authz hook -> pre-merge gate; each layer is deterministic so failure in one is caught by the next",
        "Only IDE warnings",
        "Only a chat reminder"
      ],
      "correct": 1,
      "explanation": "Multiple non-bypassable layers catch what a single layer would miss."
    },
    {
      "stage": "post",
      "question": "What does a coverage_floor check protect against?",
      "options": [
        "Hot-path latency",
        "Agents quietly deleting tests that fail; the gate fails if measured coverage drops below the floor or last merge by more than 1 percentage point",
        "Cold starts",
        "Outdated lockfiles"
      ],
      "correct": 1,
      "explanation": "Without a floor, agents can silently lower coverage to keep the verdict green."
    },
    {
      "stage": "post",
      "question": "When should --strict mode promote every warn to block?",
      "options": [
        "Always",
        "Release branches, ship-blocking PRs, post-incident triage; not the daily default because strict-on-everything corrodes flow",
        "Never",
        "Only on Sundays"
      ],
      "correct": 1,
      "explanation": "--strict is opt-in by branch; reserve for high-stakes moments."
    }
  ]
}