Refactor materials + add code for analysis and figures

.gitignore

@@ -7,6 +7,12 @@ __pycache__/
 .venv/
 venv/
 .ipynb_checkpoints/
+.pytest_cache/
+.ruff_cache/
+.coverage
+build/
+dist/
+*.egg-info/
 
 # LaTeX
 *.aux
@@ -21,6 +27,9 @@ venv/
 # Local source transcripts (not redistributed; download from HuggingFace)
 interview_transcripts/
 
+# Generated LLM runs and analysis artifacts
+outputs/
+
 # Editor
 .claude/
 .vscode/

README.md

@@ -10,19 +10,73 @@ The paper analyzes 1,250 interview transcripts from Anthropic's AI Interviewer d
 
 ```
 paper/    LaTeX source, bibliography, and figure PDFs
-code/     Analysis scripts that operate on the released data files
 coding/   Coding scheme and hand-coding protocol
 data/     LLM-coded opacity labels and hand-coded justification sheet
+configs/  LLM experiment and clustering configurations
+prompts/  Prompts used for opacity coding and task labeling
+scripts/  Batch, processing, clustering, paper-analysis, and plotting entrypoints
+src/      Installable fabricated_front Python package
+```
+
+## Installation
+
+Python 3.10 or newer is required. Using `uv`:
+
+```bash
+uv venv
+source .venv/bin/activate
+uv pip install -e .
+```
+
+## End-to-end LLM workflow
+
+Set `OPENAI_API_KEY`, then run the opacity coding pass and collect its results
+with transcripts included for the downstream task-activity pass:
+
+```bash
+uv run python scripts/submit_experiment.py configs/coding/exp0.3.yaml
+uv run python scripts/collect_batch_results.py --include-transcript configs/coding/exp0.3.yaml
+uv run python scripts/submit_experiment.py configs/task_activity/exp0.4_task_activity.yaml
+uv run python scripts/collect_batch_results.py configs/task_activity/exp0.4_task_activity.yaml
+```
+
+Cluster and label the extracted task activities:
+
+```bash
+uv run python scripts/analysis/analyze_task_activities.py
+uv run python scripts/analysis/activity_clustering.py
+```
+
+Generate the paper figures:
+
+```bash
+uv run python scripts/plots/make_justifications_figure.py
+uv run python scripts/plots/plot_all_interview_opacity.py
+uv run python scripts/plots/plot_interview_mechanism_by_type.py
+uv run python scripts/plots/plot_opacity_form.py
+uv run python scripts/plots/plot_tasks_opacity_mechanism_grid.py
+```
+
+Figure PDFs and PNGs are written to `paper/figures/`.
+
+Generated batch outputs and intermediate analysis artifacts are intentionally
+excluded from version control.
+
+## Tests
+
+```bash
+uv run --extra dev pytest
+uv run --extra dev ruff check scripts src tests
 ```
 
 ## Data
 
-The transcripts themselves are not redistributed here. They are publicly available under CC-BY at:
+The transcripts themselves are not redistributed here. The coding pipeline loads
+them directly through the Hugging Face `datasets` package. They are publicly
+available under CC-BY at:
 
 <https://huggingface.co/datasets/Anthropic/AnthropicInterviewer>
 
-Downloaded splits should be placed at `interview_transcripts/{workforce,creatives,scientists}_transcripts.csv` if you wish to re-run the coding pipeline.
-
 What is included:
 
 - `data/opacity_labeling.csv` — LLM-coded opacity labels (GPT-5.4, medium reasoning effort) for all 1,250 transcripts across five mechanisms, with per-mechanism `level`, `form`, `rationale`, and short evidence quotes. Full transcript text has been stripped; refer to the source dataset for the originals.

configs/analysis/task_activity_cluster_counts.yaml

@@ -0,0 +1,6 @@
+cluster_counts:
+  voice_opacity: 18
+  vulnerability_opacity: 12
+  provenance_opacity: 16
+  attention_opacity: 8
+  investment_opacity: 14

configs/clustering/task_activity_cluster_labels.yaml

@@ -0,0 +1,10 @@
+name: task_activity_cluster_labels_v1
+provider: openai
+model: gpt-5.4
+prompt_file: prompts/clustering/task_activity_cluster_phrase_label.txt
+reasoning_effort: medium
+input_root: artifacts/task_activity_clustering
+cluster_kind: activity
+samples_per_cluster: 12
+request:
+  max_output_tokens: 1000

configs/coding/exp0.3.yaml

@@ -0,0 +1,8 @@
+name: exp0.3
+provider: openai
+model: gpt-5.4
+prompt_file: prompts/opacity_typology_coding_v3.txt
+reasoning_effort: medium
+request:
+  temperature: 1
+  max_output_tokens: 10000

configs/task_activity/exp0.4_task_activity.yaml

@@ -0,0 +1,12 @@
+name: exp0.4_task_activity
+provider: openai
+task_type: task_activity
+model: gpt-5.4
+prompt_file: prompts/opacity_task_activity_labeling_v1.txt
+source_results_csv: outputs/runs/exp0.3/results.csv
+levels:
+  - potential
+  - clear
+reasoning_effort: medium
+request:
+  max_output_tokens: 2000

prompts/clustering/task_activity_cluster_phrase_label.txt

@@ -0,0 +1,34 @@
+You are labeling one semantic cluster of short task-activity clauses from a study of how people describe generative AI creating engagement opacity in professional settings.
+
+<context>
+Opacity mechanism: {mechanism_label}
+Mechanism blurb: {mechanism_blurb}
+
+Snippet source: {snippet_source_label}
+
+Each snippet names a concrete work activity associated with this mechanism.
+</context>
+
+<instructions>
+Read the representative activity snippets and write one short label that captures the shared type of work in the cluster.
+
+Requirements:
+- Output only the label text.
+- Use plain language.
+- Keep it to one sentence maximum.
+- Prefer a short phrase over a long explanation.
+- Focus on the activity family, not the opacity mechanism.
+- Be specific enough to distinguish the cluster from nearby clusters.
+- Do not mention "cluster", "snippet", "task activity", or the opacity mechanism name unless genuinely necessary.
+</instructions>
+
+<cluster_metadata>
+Cluster id: {cluster_id}
+Cluster size: {cluster_size}
+Level mix: {level_counts}
+Form mix: {form_counts}
+</cluster_metadata>
+
+<representative_snippets>
+{representative_snippets}
+</representative_snippets>

prompts/opacity_task_activity_labeling_v1.txt

@@ -0,0 +1,73 @@
+You are identifying the work activity in which a specific engagement opacity mechanism is manifesting.
+
+Your job is to identify the work activity or activities where the specified opacity mechanism is actually happening.
+
+Focus on the activity where the opacity is socially, organizationally, or interpersonally consequential, not just any activity where AI happens to be used.
+
+Mechanism guidance:
+- Voice opacity: identify the activity where AI shapes how the person sounds or comes across to other people.
+- Vulnerability opacity: identify the activity where AI hides uncertainty, struggle, or limits.
+- Provenance opacity: identify the activity where AI makes the basis, authorship, reasoning, or accountability of an output harder to trace or defend.
+- Attention opacity: identify the activity where AI stands in for direct attention, participation, or engagement.
+- Investment opacity: identify the activity where AI obscures how much time, labor, care, or effort went into the output.
+
+What makes a good activity label:
+- It is a short clause, not a sentence.
+- It names the concrete work activity, not the opacity mechanism.
+- It is specific enough to be meaningful in context.
+- It includes the type of work being done, not just the communication medium.
+- It stands on its own.
+
+Good examples:
+- drafting and revising emails, commits, and long-form messages
+- developing reliability analyses and strategy recommendations
+- producing audit reports and client deliverables
+
+Bad examples:
+- sending emails
+- communication
+- compliance work
+- using AI for writing
+- voice opacity in client work
+
+Rules:
+1. Focus only on the activity where the specified opacity mechanism is manifesting, not every AI use mentioned in the interview.
+2. Use the rationale and evidence to decide which activity is relevant.
+3. The opacity form tells you whether the mechanism appears in producing an output, avoiding an activity, or both. Your activity labels should fit the coded form:
+   - production: name the activity where opacity is produced
+   - avoidance: name the activity where opacity is being avoided
+   - mixed: name the closely related activity set spanning both
+4. Identify the work activity implicated by the mechanism, not the mechanism itself and not a summary of the whole interview.
+5. Prefer the most specific task label that still generalizes beyond one tiny incident.
+6. Prefer the activity where the opacity would matter to clients, coworkers, managers, audiences, or other downstream readers/users of the work.
+7. Include the type of work being done, not just the medium. For example, prefer "drafting client audit reports" over "writing documents."
+8. Return a JSON array of strings.
+9. Include multiple strings only if they are genuinely distinct activities. Do not merge distinct activities into one blended phrase.
+10. If two activities are just variants of the same workflow, combine them into one string instead of splitting them.
+11. If the interview mentions multiple AI-assisted activities, include only the activities most central to the coded mechanism.
+12. Return at most 3 strings.
+13. Do not explain your answer.
+14. Do not mention "opacity," "mechanism," "rationale," or "evidence."
+15. Do not write full sentences.
+16. Keep each string under about 12 words if possible.
+
+Mechanism: {mechanism}
+Mechanism context: {mechanism_context}
+Opacity level: {level}
+Opacity form: {form}
+
+Coding rationale:
+{rationale}
+
+Evidence:
+{evidence}
+
+Transcript:
+{transcript}
+
+Output format:
+Return only a JSON array of strings.
+
+Example valid outputs:
+["drafting professional emails"]
+["drafting professional emails", "writing goodbye emails to colleagues"]

prompts/opacity_typology_coding_v3.txt

@@ -0,0 +1,149 @@
+You are a careful qualitative research assistant analyzing workplace interview transcripts about AI use.
+
+Your task is to classify each transcript according to the typology of engagement opacity defined below. Use only evidence from the transcript. Do not infer beyond what is reasonably supported by the text.
+
+Conceptual frame:
+Generative AI can create engagement opacity by decoupling observable output from human engagement. In workplace interaction, this can weaken trust, accountability, and coordination by making it harder for one person to interpret another person's attention, authorship, commitments, competence, effort, or care.
+
+The key analytic rule is that opacity is interpersonal. The relevant question is not just whether someone uses AI for a certain activity, but whether that activity creates or could create ambiguity, misrecognition, or distorted inference between humans in workplace interaction.
+
+Classify the transcript using these five opacity mechanisms:
+
+1. Voice opacity
+Definition:
+Voice opacity occurs when GenAI filters or standardizes stance cues and footing, making it hard for other people to tell whether the speaker is actually authoring and standing behind the communication or merely animating polished text. It reduces personhood cues such as distinctive phrasing, tonal habits, humor, and other relational markers through which coworkers recognize one another and coordinate expectations.
+
+Look for:
+- using AI to draft or rewrite messages in a polished or standardized tone
+- loss of personal style, voice, or individuality in communication with others
+- difficulty signaling sincerity, ownership, or commitment to coworkers or clients
+- concern that communication no longer reflects the actual speaker
+
+2. Vulnerability opacity
+Definition:
+Vulnerability opacity occurs when AI allows workers to present an appearance of immediate competence or fluency while hiding uncertainty, confusion, struggle, or lack of knowledge from other people. This suppresses the interpersonal signals through which coworkers offer help, mentorship, or relational support.
+
+Look for:
+- using AI to avoid appearing confused, unprepared, or inexperienced to others
+- masking uncertainty, hesitation, or lack of understanding in human interaction
+- reduced opportunities for asking for help, learning, mentoring, or bonding through shared difficulty
+- concern that coworkers cannot see when someone is struggling
+
+3. Provenance opacity
+Definition:
+Provenance opacity occurs when a worker produces an artifact that appears owned and defensible to other people, but cannot fully explain how or why it was produced, what assumptions shaped it, or how to repair it without returning to the AI system. The output appears attributable, but its reasoning chain is not fully reconstructable by the user when others demand explanation or accountability.
+
+Look for:
+- inability to explain, defend, or revise AI-generated work to others
+- dependence on re-prompting rather than understanding when others ask questions
+- uncertainty about where ideas, wording, structure, or decisions came from in a shared work product
+- concern about accountability for artifacts produced with AI assistance
+
+4. Attention opacity
+Definition:
+Attention opacity occurs when interactional participation is decoupled from actual engagement. Workers can appear present, informed, or participatory to other people while delegating attention to AI tools such as summarizers or note-takers. This can also reshape the "official memory" of meetings by collapsing nuance, disagreement, or multiple perspectives into a single flattened account that others may treat as authoritative.
+
+Look for:
+- use of AI summaries instead of direct participation or attention in interactions with others
+- appearing engaged without actually attending closely
+- relying on AI-generated notes, summaries, or recaps in place of firsthand engagement
+- concern that AI summaries erase nuance, dissent, ambiguity, or multiple perspectives for other participants
+
+5. Investment opacity
+Definition:
+Investment opacity occurs when high-effort and low-effort outputs become difficult for other people to distinguish because AI enables polished results with reduced visible labor. This weakens the basis for assigning credit, trust, responsibility, or judgments of diligence and care.
+
+Look for:
+- concern that effort is no longer visible to coworkers, managers, clients, or collaborators
+- polished outputs produced with minimal investment but interpreted by others as high effort
+- difficulty evaluating care, diligence, seriousness, or commitment in relation to other people
+- changes in credit allocation, trust, or evaluation because effort signals are obscured
+
+Coding scale:
+For each mechanism, assign exactly one of these three levels:
+- "none": no meaningful evidence of this opacity mechanism
+- "potential": the speaker describes an AI-supported practice that could create this opacity between humans, but does not clearly describe the interpersonal opacity effect itself
+- "clear": the speaker explicitly describes the interpersonal opacity, accountability gap, trust distortion, or engagement ambiguity between people
+
+Examples of the distinction:
+- "I use AI-generated meeting summaries" by itself is usually "potential" attention opacity.
+- "I use AI-generated meeting summaries and other people assume I was paying attention when I was not" is "clear" attention opacity.
+- "I use AI to help draft emails" by itself is usually "potential" voice opacity.
+- "I use AI to draft emails and now coworkers cannot tell what I actually think or whether I stand behind the message" is "clear" voice opacity.
+
+General coding instructions:
+- Code only what is supported by the transcript.
+- A mechanism may be relevant even if the interviewee does not use this terminology directly.
+- Multiple mechanisms may appear in the same transcript.
+- Do not classify ordinary productivity benefits or generic AI enthusiasm unless they specifically relate to interpersonal opacity in workplace interaction.
+- Do not rely on style alone; use the substance of what the speaker says about workplace interaction, coordination, trust, evaluation, communication, or accountability.
+- Tool use alone is not enough for "clear". "Clear" requires explicit evidence of how other people interpret, misinterpret, trust, evaluate, or coordinate around the AI-mediated practice.
+- When in doubt between "none" and "potential", choose "potential" only if the transcript clearly describes the relevant AI-supported practice.
+- When in doubt between "potential" and "clear", choose "potential" unless the interpersonal opacity effect is explicit.
+
+Additional coding rule:
+A transcript may discuss an opacity mechanism by describing its production, its avoidance, or both. In other words, the mechanism may be relevant not only when the speaker is creating, reinforcing, or reproducing that opacity mechanism, but also when the speaker is explicitly trying to prevent, limit, or work around it.
+
+For each mechanism, code both:
+- level: how strongly the transcript evidences the mechanism
+- form: whether the mechanism appears here as production, avoidance, mixed, or none
+
+Form scale:
+For each mechanism, assign exactly one of these four forms:
+- "production": the speaker describes practices that create, reinforce, or reproduce this opacity mechanism
+- "avoidance": the speaker describes trying to prevent, limit, or work around this opacity mechanism
+- "mixed": the transcript includes both production and avoidance of this mechanism
+- "none": the mechanism is not meaningfully in play, so no form applies
+
+Output instructions:
+Return valid JSON only. No markdown. No prose before or after the JSON.
+
+Use this exact schema:
+
+{
+  "summary": "2-4 sentence analytic summary of the transcript focused on engagement opacity in human-to-human workplace interaction.",
+  "mechanisms": {
+    "voice_opacity": {
+      "level": "none",
+      "form": "none",
+      "evidence": ["quote or close paraphrase 1", "quote or close paraphrase 2"],
+      "rationale": "brief explanation"
+    },
+    "vulnerability_opacity": {
+      "level": "potential",
+      "form": "production",
+      "evidence": ["quote or close paraphrase 1"],
+      "rationale": "brief explanation"
+    },
+    "provenance_opacity": {
+      "level": "clear",
+      "form": "production",
+      "evidence": ["quote or close paraphrase 1"],
+      "rationale": "brief explanation"
+    },
+    "attention_opacity": {
+      "level": "none",
+      "form": "none",
+      "evidence": ["quote or close paraphrase 1"],
+      "rationale": "brief explanation"
+    },
+    "investment_opacity": {
+      "level": "none",
+      "form": "none",
+      "evidence": ["quote or close paraphrase 1"],
+      "rationale": "brief explanation"
+    }
+  }
+}
+
+Important:
+- Every mechanism must be included.
+- Each mechanism must use exactly one level: "none", "potential", or "clear".
+- Each mechanism must use exactly one form: "production", "avoidance", "mixed", or "none".
+- If a mechanism is "none", provide an empty evidence array if needed and explain briefly why evidence is insufficient.
+- If level is "none", form should usually be "none".
+- If level is "potential" or "clear", form should be "production", "avoidance", or "mixed".
+- Base all judgments on the transcript alone.
+- A speaker can treat a mechanism as a real concern even when they are resisting it.
+- For example, if someone says AI-generated writing loses the personal touch and they therefore avoid using it in sensitive communication, that is evidence of voice opacity in the form of avoidance rather than production.
+- "Avoidance" means the transcript shows the speaker actively preventing or limiting that opacity mechanism, not merely expressing general dislike of AI.