Scaffold behavioral evals in 'pc init'

CHANGELOG.md

@@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `pc init` can now scaffold **behavioral evals** (Pipecat's `pipecat.evals`
+  framework, pipecat-ai/pipecat#4655). For cascade bots on standard transports the
+  wizard asks "Include behavioral evals?" (default yes); non-interactive runs use
+  `--evals/--no-evals` or an `"evals"` key in `--config` JSON. When enabled, the
+  generated bot exposes an `eval` transport (`uv run bot.py -t eval`), the project
+  ships `server/evals/scenario.yaml` (judge: OpenAI when the bot's LLM is OpenAI,
+  otherwise local Ollama), and the pipecat dependency gains the `cli` extra so
+  `uv run pipecat eval run evals/scenario.yaml` works. Requires the pipecat
+  release after 1.3.0.
+
 - `pc init` now accepts an optional target directory. Passing a path — for
   example `pc init .` — scaffolds the project **directly into that directory**
   instead of nesting it under a `<project-name>/` subfolder (the same convention

scripts/imports/import_generator.py

@@ -372,6 +372,9 @@ def _get_external_module_path(class_name: str) -> str | None:
     # Explicit so regeneration doesn't depend on the searched source tree containing it
     # — see the "create_transport" feature. The collapsed bot() imports create_transport.
     "create_transport": "pipecat.runner.utils",
+    # The eval transport entry uses WebsocketServerParams from the websocket *server*
+    # module (NOT .fastapi, which the 'websocket'/telephony transports use).
+    "WebsocketServerParams": "pipecat.transports.websocket.server",
 }
 
 

src/pipecat_cli/commands/init.py

@@ -110,6 +110,12 @@ def init_command(
     observability: bool = typer.Option(
         False, "--observability/--no-observability", help="Enable observability"
     ),
+    evals: bool = typer.Option(
+        False,
+        "--evals/--no-evals",
+        help="Include behavioral evals (eval transport + scenario; cascade mode with "
+        "standard transports only)",
+    ),
     config: Path | None = typer.Option(
         None, "--config", "-c", help="JSON config file (triggers non-interactive mode)"
     ),
@@ -208,6 +214,7 @@ def init_command(
                 observability = observability or file_data.get(
                     "observability", file_data.get("enable_observability", False)
                 )
+                evals = evals or file_data.get("evals", file_data.get("enable_evals", False))
 
             try:
                 project_config = validate_and_build_config(
@@ -231,6 +238,7 @@ def init_command(
                     deploy_to_cloud=deploy_to_cloud,
                     enable_krisp=enable_krisp,
                     observability=observability,
+                    evals=evals,
                 )
             except ConfigValidationError as e:
                 console.print(f"\n[red]{e}[/red]")

src/pipecat_cli/config_validator.py

@@ -9,7 +9,7 @@
 import json
 from pathlib import Path
 
-from pipecat_cli.prompts.questions import ProjectConfig
+from pipecat_cli.prompts.questions import EVALS_ELIGIBLE_TRANSPORTS, ProjectConfig
 from pipecat_cli.registry import ServiceRegistry
 
 
@@ -52,6 +52,7 @@ def validate_and_build_config(
     deploy_to_cloud: bool = True,
     enable_krisp: bool = False,
     observability: bool = False,
+    evals: bool = False,
 ) -> ProjectConfig:
     """Validate all inputs and build a ProjectConfig.
 
@@ -232,6 +233,15 @@ def validate_and_build_config(
         errors.append("--video-output is only available for web bots")
     if enable_krisp and not deploy_to_cloud:
         errors.append("--enable-krisp requires --deploy-to-cloud")
+    if evals:
+        if mode == "realtime":
+            errors.append("--evals is only available in cascade mode")
+        ineligible = [t for t in resolved_transports if t not in EVALS_ELIGIBLE_TRANSPORTS]
+        if ineligible:
+            errors.append(
+                f"--evals is not supported with transport(s): {', '.join(ineligible)}. "
+                f"Supported: {', '.join(sorted(EVALS_ELIGIBLE_TRANSPORTS))}"
+            )
 
     # --- Daily PSTN mode without matching transport ---
     if daily_pstn_mode and transport and "daily_pstn" not in transport:
@@ -290,6 +300,7 @@ def validate_and_build_config(
         deploy_to_cloud=deploy_to_cloud,
         enable_krisp=enable_krisp,
         enable_observability=observability,
+        enable_evals=evals,
     )
     return config
 
@@ -329,5 +340,6 @@ def config_to_json(config: ProjectConfig) -> str:
         "deploy_to_cloud": config.deploy_to_cloud,
         "enable_krisp": config.enable_krisp,
         "enable_observability": config.enable_observability,
+        "enable_evals": config.enable_evals,
     }
     return json.dumps(data, indent=2)

src/pipecat_cli/generators/project.py

@@ -167,6 +167,10 @@ def _write_project_files(self, project_path: Path) -> Path:
         # 3. Generate .env.example (in server/)
         self._generate_env_example(server_path)
 
+        # 3b. Generate evals/scenario.yaml (in server/, if enabled)
+        if self.config.enable_evals:
+            self._generate_evals(server_path)
+
         # 4. Generate .gitignore (at root)
         self._generate_gitignore(project_path)
 
@@ -285,6 +289,7 @@ def _generate_bot_file(self, project_path: Path) -> None:
             "recording": self.config.recording,
             "transcription": self.config.transcription,
             "observability": self.config.enable_observability,
+            "evals": self.config.enable_evals,
         }
 
         # Get imports
@@ -315,6 +320,7 @@ def _generate_bot_file(self, project_path: Path) -> None:
             "transcription": self.config.transcription,
             "enable_krisp": self.config.enable_krisp,
             "enable_observability": self.config.enable_observability,
+            "enable_evals": self.config.enable_evals,
             "service_configs": ServiceRegistry.SERVICE_CONFIGS,
             "daily_pstn_mode": self.config.daily_pstn_mode,
             "twilio_daily_sip_mode": self.config.twilio_daily_sip_mode,
@@ -347,6 +353,11 @@ def _generate_pyproject(self, project_path: Path) -> None:
         # Extract all required extras
         extras = ServiceLoader.extract_extras_for_services(services)
 
+        if self.config.enable_evals:
+            # `pipecat eval` (and python -m pipecat.evals) imports typer/rich from the
+            # cli extra.
+            extras.add("cli")
+
         # Build the pipecat-ai dependency string
         # No version constraint - will use latest from PyPI
         pipecat_extras = ",".join(sorted(extras))
@@ -382,6 +393,18 @@ def _generate_env_example(self, project_path: Path) -> None:
         content = template.render(**context)
         (project_path / ".env.example").write_text(content, encoding="utf-8")
 
+    def _generate_evals(self, project_path: Path) -> None:
+        """Generate evals/scenario.yaml for the behavioral eval harness."""
+        template = self.env.get_template("server/evals/scenario.yaml.jinja2")
+
+        content = template.render(
+            project_name=self.config.project_name,
+            llm_service=self.config.llm_service,
+        )
+        evals_dir = project_path / "evals"
+        evals_dir.mkdir(exist_ok=True)
+        (evals_dir / "scenario.yaml").write_text(content, encoding="utf-8")
+
     def _generate_gitignore(self, project_path: Path) -> None:
         """Generate .gitignore file."""
         template = self.env.get_template("gitignore.jinja2")
@@ -442,6 +465,7 @@ def _generate_readme(self, project_path: Path) -> None:
             "transcription": self.config.transcription,
             "enable_krisp": self.config.enable_krisp,
             "enable_observability": self.config.enable_observability,
+            "enable_evals": self.config.enable_evals,
             "deploy_to_cloud": self.config.deploy_to_cloud,
             "generate_client": self.config.generate_client,
             "client_framework": self.config.client_framework,
@@ -552,6 +576,11 @@ def print_next_steps(self, project_path: Path, in_place: bool = False) -> None:
         has_telephony = any(t in telephony_transports for t in self.config.transports)
 
         console.print("  • Run your bot: [bold cyan]uv run bot.py[/bold cyan]")
+        if self.config.enable_evals:
+            console.print(
+                "  • Run evals: [bold cyan]uv run bot.py -t eval[/bold cyan], then in another "
+                "terminal [bold cyan]uv run pipecat eval run evals/scenario.yaml[/bold cyan]"
+            )
         if has_telephony:
             console.print(
                 "  • Expose it for telephony: [bold cyan]ngrok http 7860[/bold cyan], then point "

src/pipecat_cli/prompts/questions.py

@@ -57,6 +57,29 @@ def replace_question_with_answer(question: str, answer: str | list[str]):
     console.print(f"[green]✔[/green] {question} [cyan]{answer_str}[/cyan]")
 
 
+# Transports that support behavioral evals: the unified create_transport path,
+# where the generated bot can expose a transport_params["eval"] entry. The
+# realtime mode and the Daily PSTN / Twilio+Daily SIP bespoke flows are excluded.
+EVALS_ELIGIBLE_TRANSPORTS = {
+    "daily",
+    "smallwebrtc",
+    "websocket",
+    "twilio",
+    "telnyx",
+    "plivo",
+    "exotel",
+}
+
+
+def evals_eligible(mode: str | None, transports: list[str]) -> bool:
+    """Whether behavioral evals can be scaffolded for this mode/transport combo."""
+    return (
+        mode == "cascade"
+        and bool(transports)
+        and all(t in EVALS_ELIGIBLE_TRANSPORTS for t in transports)
+    )
+
+
 @dataclass
 class ProjectConfig:
     """Configuration for a Pipecat project."""
@@ -106,6 +129,9 @@ class ProjectConfig:
     # Observability
     enable_observability: bool = False
 
+    # Behavioral evals (cascade mode + standard transports only)
+    enable_evals: bool = False
+
 
 def ask_project_questions(default_name: str | None = None) -> ProjectConfig:
     """
@@ -484,6 +510,7 @@ def ask_project_questions(default_name: str | None = None) -> ProjectConfig:
         replace_question_with_answer("Realtime service:", realtime_label)
 
     # Question 6: Feature customization gate
+    can_use_evals = evals_eligible(config.mode, config.transports)
     console.print("\n[bold]Default feature settings:[/bold]")
     console.print("  • Audio recording: [dim]No[/dim]")
     console.print("  • Transcription logging: [dim]No[/dim]")
@@ -494,6 +521,8 @@ def ask_project_questions(default_name: str | None = None) -> ProjectConfig:
         console.print("  • Video input: [dim]No[/dim]")
         console.print("  • Video output: [dim]No[/dim]")
     console.print("  • Observability: [dim]No[/dim]")
+    if can_use_evals:
+        console.print("  • Behavioral evals: [green]Yes[/green] [dim](recommended)[/dim]")
 
     customize_features = questionary.confirm(
         "Customize feature settings?",
@@ -598,6 +627,17 @@ def ask_project_questions(default_name: str | None = None) -> ProjectConfig:
         replace_question_with_answer(
             "Enable observability?", "Yes" if config.enable_observability else "No"
         )
+
+        # Question 6h: Behavioral evals (cascade + standard transports only)
+        if can_use_evals:
+            config.enable_evals = questionary.confirm(
+                "Include behavioral evals?",
+                default=True,
+                style=custom_style,
+            ).ask()
+            replace_question_with_answer(
+                "Include behavioral evals?", "Yes" if config.enable_evals else "No"
+            )
     else:
         # Apply default feature settings
         config.video_service = None
@@ -606,6 +646,7 @@ def ask_project_questions(default_name: str | None = None) -> ProjectConfig:
         config.recording = False
         config.transcription = False
         config.enable_observability = False
+        config.enable_evals = can_use_evals
 
     # Question 7: Pipecat Cloud deployment
     config.deploy_to_cloud = questionary.confirm(

src/pipecat_cli/registry/_imports.py

@@ -216,6 +216,7 @@
         "from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies"
     ],
     "create_transport": ["from pipecat.runner.utils import create_transport"],
+    "evals": ["from pipecat.transports.websocket.server import WebsocketServerParams"],
 }
 
 # Base imports always included in generated bot files

src/pipecat_cli/registry/service_loader.py

@@ -268,6 +268,8 @@ def get_imports_for_services(
             imports.update(ServiceRegistry.FEATURE_IMPORTS["transcription"])
         if features.get("observability"):
             imports.update(ServiceRegistry.FEATURE_IMPORTS["observability"])
+        if features.get("evals"):
+            imports.update(ServiceRegistry.FEATURE_IMPORTS["evals"])
 
         # Most bots build transports via create_transport, so import it whenever the
         # bot uses that collapsed path. Only dial-out and SIP keep a bespoke flow that

src/pipecat_cli/registry/service_metadata.py

@@ -113,6 +113,9 @@ def __post_init__(self):
     # Imported on the standard (non-PSTN/SIP) transport path: the collapsed bot()
     # calls create_transport. Dial-out and SIP construct their transports by hand.
     "create_transport": ["create_transport"],
+    # Behavioral evals: the generated bot exposes `-t eval` via an explicit
+    # transport_params["eval"] -> WebsocketServerParams entry (pipecat-ai/pipecat#4655).
+    "evals": ["WebsocketServerParams"],
 }
 
 

src/pipecat_cli/templates/README.md.jinja2

@@ -233,6 +233,10 @@ With Tail you can:
    ```bash
    pipecat tail
    ```
+{% endif %}
+{% if enable_evals %}
+{% include '_readme_blocks/evals.jinja2' %}
+
 {% endif %}
 ## Learn More
 

src/pipecat_cli/templates/_readme_blocks/evals.jinja2

@@ -0,0 +1,28 @@
+## Testing your bot (evals)
+
+This project includes a behavioral eval scenario (`server/evals/scenario.yaml`). It
+drives a text conversation with your bot and uses an LLM judge to check the replies.
+
+1. **Run the bot with the eval transport** (terminal 1, from `server/`):
+
+   ```bash
+   uv run bot.py -t eval
+   ```
+
+2. **Run the scenario** (terminal 2, from `server/`):
+
+   ```bash
+   uv run pipecat eval run evals/scenario.yaml
+   ```
+
+{% if llm_service in ['openai_llm', 'openai_responses_llm'] %}
+The judge uses OpenAI (`OPENAI_API_KEY` from your `.env`).
+{% else %}
+The judge runs locally via [Ollama](https://ollama.com). Install it and pull the
+model first (`ollama pull gemma2:9b`), or switch the scenario's judge to OpenAI.
+{% endif %}
+
+Edit `evals/scenario.yaml` to add turns and checks for your own use case. See the
+[Pipecat evals docs](https://docs.pipecat.ai) for the full scenario format,
+including audio-mode scenarios and function call assertions.
+

src/pipecat_cli/templates/server/bot_cascade.py.jinja2

@@ -140,6 +140,13 @@ async def bot(runner_args: RunnerArguments):
             audio_out_enabled=True,
         ),
         {% endfor %}
+        {% if enable_evals %}
+        # Behavioral evals: run `uv run bot.py -t eval`, then `pipecat eval run` connects here.
+        "eval": lambda: WebsocketServerParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+        ),
+        {% endif %}
     }
 
     transport = await create_transport(runner_args, transport_params)

src/pipecat_cli/templates/server/evals/scenario.yaml.jinja2

@@ -0,0 +1,38 @@
+# Behavioral eval scenario for {{ project_name }}.
+#
+# Run it in two terminals (both from server/):
+#   1. uv run bot.py -t eval
+#   2. uv run pipecat eval run evals/scenario.yaml
+#
+# Each turn optionally sends a `user:` message, then asserts `expect:` events in
+# order (response, llm_started, llm_response, function_call, user_transcription).
+# Expectations support `within_ms` (latency budget), `text_contains` (substring),
+# and `eval` (a natural-language criterion graded by the judge LLM below).
+name: {{ project_name | replace('-', '_') }}_smoke
+
+judge:
+  eval:
+{% if llm_service in ['openai_llm', 'openai_responses_llm'] %}
+    service: openai  # uses OPENAI_API_KEY from your .env
+    model: gpt-4o
+    # Or run the judge locally with Ollama (https://ollama.com, then `ollama pull gemma2:9b`):
+    # service: ollama
+    # model: gemma2:9b
+{% else %}
+    service: ollama  # local judge: install Ollama (https://ollama.com), then `ollama pull gemma2:9b`
+    model: gemma2:9b
+    # Or use OpenAI as the judge (requires OPENAI_API_KEY in your .env):
+    # service: openai
+    # model: gpt-4o
+{% endif %}
+
+turns:
+  # Wait for the bot's on-connect greeting before speaking.
+  - expect:
+      - event: response
+        eval: "the bot opens the conversation in some way (a greeting, an introduction, or an offer to help)"
+
+  - user: "What is the capital of France?"
+    expect:
+      - event: response
+        eval: "the response says the capital of France is Paris"