|
18 | 18 | Note the outline template receives `speakers` but NOT `speaker_names`. |
19 | 19 | """ |
20 | 20 |
|
| 21 | +import json |
21 | 22 | import re |
22 | 23 | from pathlib import Path |
23 | 24 |
|
|
40 | 41 | ] |
41 | 42 |
|
42 | 43 | # Strings that must never reach the model: each one is copyable as content. |
| 44 | +# Angle-bracket descriptions count too - a model that copies |
| 45 | +# "<the complete words this speaker says out loud>" into a dialogue value sends |
| 46 | +# that straight to the TTS engine, and the templates' own rules ban placeholders. |
43 | 47 | COPYABLE_SKELETONS = ( |
44 | 48 | "[Actual Speaker Name]", |
45 | 49 | "[Speaker's dialogue based on their personality and expertise]", |
46 | 50 | "[Segment Name]", |
47 | 51 | "[Description of the segment content]", |
48 | 52 | '{"transcript": [...]}', |
49 | 53 | '{"segments": [...]}', |
| 54 | + "<the complete words this speaker says out loud, written out in full>", |
| 55 | + "<the real title of this segment>", |
| 56 | + "<the real title of the next segment>", |
| 57 | + "<what is discussed in this segment, including the key points and questions to cover>", |
| 58 | + "<what is discussed in that segment, including the key points and questions to cover>", |
50 | 59 | ) |
51 | 60 |
|
52 | 61 |
|
@@ -87,6 +96,59 @@ def render_outline(**overrides) -> str: |
87 | 96 | return render("outline", **data) |
88 | 97 |
|
89 | 98 |
|
| 99 | +def example_object(rendered: str, root_key: str) -> dict: |
| 100 | + """Parse the JSON example the prompt shows the model. |
| 101 | +
|
| 102 | + The example is the contract the model imitates, so it has to be valid JSON |
| 103 | + in its own right - a speaker name carrying a quote or a backslash would |
| 104 | + otherwise hand the model a broken example to copy. |
| 105 | + """ |
| 106 | + prefix = '{"' + root_key + '":' |
| 107 | + for line in rendered.splitlines(): |
| 108 | + if line.startswith(prefix): |
| 109 | + return json.loads(line) |
| 110 | + raise AssertionError(f"no {root_key} example found in the rendered prompt") |
| 111 | + |
| 112 | + |
| 113 | +class TestExampleIsValidAndComplete: |
| 114 | + """Whatever the model copies from the example must be usable output.""" |
| 115 | + |
| 116 | + def test_transcript_example_parses_and_names_the_speakers(self): |
| 117 | + example = example_object(render_transcript(), "transcript") |
| 118 | + assert [entry["speaker"] for entry in example["transcript"]] == [ |
| 119 | + "Marcus Thompson", |
| 120 | + "Elena Vasquez", |
| 121 | + ] |
| 122 | + |
| 123 | + def test_transcript_example_survives_json_special_characters(self): |
| 124 | + r"""A name like Dr. "Alex" Chen\ must be escaped, not interpolated raw.""" |
| 125 | + speakers = [ |
| 126 | + {"name": 'Dr. "Alex" Chen\\', "backstory": "b", "personality": "p"}, |
| 127 | + {"name": "Jamie\tRodriguez", "backstory": "b", "personality": "p"}, |
| 128 | + ] |
| 129 | + example = example_object(render_transcript(speakers=speakers), "transcript") |
| 130 | + assert [entry["speaker"] for entry in example["transcript"]] == [ |
| 131 | + 'Dr. "Alex" Chen\\', |
| 132 | + "Jamie\tRodriguez", |
| 133 | + ] |
| 134 | + |
| 135 | + def test_transcript_example_dialogue_is_speakable(self): |
| 136 | + """Dialogue goes straight to TTS, so the example must not contain a |
| 137 | + description of what to write - a copied one would be read aloud.""" |
| 138 | + example = example_object(render_transcript(), "transcript") |
| 139 | + for entry in example["transcript"]: |
| 140 | + assert "<" not in entry["dialogue"] |
| 141 | + assert entry["dialogue"].endswith(".") |
| 142 | + |
| 143 | + def test_outline_example_parses_with_valid_sizes(self): |
| 144 | + example = example_object(render_outline(), "segments") |
| 145 | + assert example["segments"] |
| 146 | + for segment in example["segments"]: |
| 147 | + assert segment["size"] in {"short", "medium", "long"} |
| 148 | + assert "<" not in segment["name"] |
| 149 | + assert "<" not in segment["description"] |
| 150 | + |
| 151 | + |
90 | 152 | class TestNoCopyableSkeletons: |
91 | 153 | """Whatever the model copies from the prompt must be valid output.""" |
92 | 154 |
|
@@ -154,6 +216,14 @@ def test_outline_includes_the_language_instruction(self): |
154 | 216 | assert "IMPORTANT LANGUAGE INSTRUCTION" in rendered |
155 | 217 | assert "segment names, descriptions" in rendered |
156 | 218 |
|
| 219 | + @pytest.mark.parametrize( |
| 220 | + "renderer", [render_transcript, render_outline], ids=["transcript", "outline"] |
| 221 | + ) |
| 222 | + def test_english_sample_is_flagged_as_english(self, renderer): |
| 223 | + """The example is hard-coded English; say so, or it nudges the model |
| 224 | + back toward English for a non-English episode.""" |
| 225 | + assert "in English only to show the structure" in renderer(language="Hebrew") |
| 226 | + |
157 | 227 | @pytest.mark.parametrize( |
158 | 228 | "renderer", [render_transcript, render_outline], ids=["transcript", "outline"] |
159 | 229 | ) |
|
0 commit comments