docassemble-ALToolbox/promptfoo-goal-oriented-questions.yml at main · SuffolkLITLab/docassemble-ALToolbox · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# promptfoo.yaml for GoalOrientedQuestionList Initial Prompt Evaluation

prompts:
  - id: goalorientedquestion-initial
    raw: |
      You are a legal aid intake worker creating an intake form.
      Based on this question, generate between 1 and 3 structured fields to gather the initial information:

      Question: {{question}}

      Goal: {{goal}}

      Respond with a JSON object in this format:
      {
        "question_text": "Brief intro text for the question screen",
        "fields": [
          {
            "label": "Field label text",
            "field": "temp_field_1",
            "datatype": "yesnoradio|radio|checkboxes|text|area|date|currency|email",
            "choices": ["Option 1", "Option 2"],  // only for radio/checkboxes
            "required": false
          }
        ]
      }

      Guidelines:
      - Generate 1–3 specific fields that help gather information relevant to the rubric
      - Use yesnoradio for yes/no questions
      - Use radio for single-choice questions (2–5 options)
      - Use checkboxes for multiple-choice questions
      - Use text for short text responses
      - Use area for longer narrative responses
      - Use date only when a precise date is likely to be known
      - All fields must have required: false
      - Write questions and field labels at about a 6th-grade reading level
      - Ask one question per field, avoiding compound questions
      - Prefer structured question types over open-ended ones

providers:
  - id: openai:gpt-5-nano
    label: GPT-5 Nano (reasoning=minimal)
    config:
      reasoning_effort: minimal

  - id: openai:gpt-5-nano
    label: GPT-5 Nano (reasoning=low)
    config:
      reasoning_effort: low

  - id: openai:gpt-5-nano
    label: GPT-5 Nano (reasoning=medium)
    config:
      reasoning_effort: medium

  - id: openai:gpt-4.1-nano
    label: GPT-4.1 Nano (default)

  - id: openai:gpt-5-mini
    label: GPT-5 Mini (reasoning=minimal)
    config:
      reasoning_effort: minimal

  - id: openai:gpt-5-mini
    label: GPT-5 Mini (reasoning=low)
    config:
      reasoning_effort: low

tests:
  - id: housing-situation
    vars:
      question: We need to understand your housing situation to determine how we can help. Please provide information about your housing issue.
      goal: The response provides complete information about the tenant's housing situation, including the type of issue, timeline, communication with landlord, and any immediate safety or habitability concerns.

  - id: eviction-notice
    vars:
      question: Have you received any eviction or termination notices? If so, please describe.
      goal: The response identifies whether the client has received an eviction notice, when it was received, and from whom.

  - id: rent-payment-issues
    vars:
      question: Are you currently behind on rent or facing difficulties paying rent?
      goal: The response captures information about rent arrears, reasons for nonpayment, and any payment plans or negotiations attempted.
  - id: family-children
    vars:
      question: Are there any children involved in this case?
      goal: The response clarifies how many children, their ages, and any existing custody or visitation arrangements.

  - id: family-safety
    vars:
      question: Have there been any incidents of domestic violence, threats, or safety concerns?
      goal: The response identifies any immediate safety issues that could require protective orders or referrals.

  - id: family-finances
    vars:
      question: What is your current income and employment situation?
      goal: The response captures income, employment, and dependents information necessary for support calculations or fee waivers.

metrics:
  - name: latency

evals:
  - type: llm-rubric
    criteria:
      - "Are the questions close ended when possible?"
      - "Are the generated questions relevant to the intake goal?"
      - "Are the questions easy to understand?"

output:
  format: table