agent-engineering-handbook/skills/iam-security-advisor/evals/evals.json at main · d-padmanabhan/agent-engineering-handbook · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
{
  "skill_name": "iam-security-advisor",
  "evals": [
    {
      "id": 1,
      "prompt": "Choose between a long-lived cloud access key stored in GitHub Actions secrets and OIDC workload federation for production deployments. Recommend one architecture, identify the trust conditions, explain failure and revocation behavior, and state what facts would reverse the decision.",
      "expected_output": "Recommend short-lived OIDC workload federation, constrain issuer/audience/subject and authorization, cover token replay, revocation and audit, and provide explicit reversal conditions.",
      "files": [],
      "checks": [
        {
          "id": "recommends-federation",
          "type": "contains_all",
          "values": [
            "OIDC",
            "short-lived"
          ]
        },
        {
          "id": "trust-conditions",
          "type": "contains_all",
          "values": [
            "issuer",
            "audience",
            "subject"
          ]
        },
        {
          "id": "lifecycle-and-evidence",
          "type": "contains_all",
          "values": [
            "revocation",
            "audit"
          ]
        },
        {
          "id": "reversal-conditions",
          "type": "contains_any",
          "values": [
            "reversal conditions",
            "would change the recommendation",
            "reverse this decision"
          ]
        }
      ]
    },
    {
      "id": 2,
      "prompt": "Give a security architecture deep dive for an AI agent using OAuth delegation to call a customer records API on behalf of a user. Trace the actors and trust boundaries, separate authentication from authorization, and analyze token misuse and recovery.",
      "expected_output": "Enumerate the user, agent/client, authorization server, resource server, and policy enforcement point; trace issuance and delegation; cover audience, scope, lifetime, replay, confused deputy, revocation, and audit correlation.",
      "files": [],
      "checks": [
        {
          "id": "actors",
          "type": "contains_all",
          "values": [
            "authorization server",
            "resource server",
            "user",
            "agent"
          ]
        },
        {
          "id": "token-contract",
          "type": "contains_all",
          "values": [
            "audience",
            "scope",
            "lifetime"
          ]
        },
        {
          "id": "attack-paths",
          "type": "contains_all",
          "values": [
            "replay",
            "confused deputy",
            "token theft"
          ]
        },
        {
          "id": "recovery",
          "type": "contains_all",
          "values": [
            "revocation",
            "audit"
          ]
        }
      ]
    },
    {
      "id": 3,
      "prompt": "A Kubernetes workload needs a database credential. Compare environment-variable injection, a mounted secret, and workload-identity-based retrieval. Start by deciding whether the static secret can be removed. Recommend one approach and include outage behavior, rotation, exposure, recovery, and reversal conditions.",
      "expected_output": "First consider eliminating the secret with workload identity. If a secret remains, compare delivery mechanisms across freshness, exposure, audit, outage coupling, revocation, and recovery, then recommend one with reversal conditions.",
      "files": [],
      "checks": [
        {
          "id": "eliminate-secret-first",
          "type": "contains_any",
          "values": [
            "eliminate the secret",
            "remove the static secret",
            "workload identity"
          ]
        },
        {
          "id": "rejects-env-default",
          "type": "contains_any",
          "values": [
            "avoid environment variables",
            "not recommend environment",
            "environment variables expose"
          ]
        },
        {
          "id": "lifecycle",
          "type": "contains_all",
          "values": [
            "rotation",
            "revocation",
            "outage",
            "recovery"
          ]
        },
        {
          "id": "decisive",
          "type": "contains_any",
          "values": [
            "recommend",
            "preferred approach",
            "choose"
          ]
        }
      ]
    }
  ]
}