feat(voice): infer expressive speech register

.changeset/register-inference.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Replace expressive speech presets with register inference and sparse non-verbal opt-outs.

agents/src/tts/expr_markup.test.ts

@@ -12,6 +12,7 @@
  * provider's instruction block advertises only what that provider supports.
  */
 import { describe, expect, it } from 'vitest';
+import { DEFAULT_EXPRESSIVE_OPTIONS, resolveExpressiveOptions } from '../voice/agent_session.js';
 import {
   TranscriptMarkupStripper,
   convertMarkup,
@@ -20,6 +21,7 @@ import {
   normalizeMarkup,
   splitAllMarkup,
   splitMarkup,
+  steeringInstructions,
   supportedNonverbals,
 } from './_provider_format.js';
 
@@ -193,13 +195,15 @@ describe('Fish Audio dialect', () => {
   });
 
   it('filters sounds and examples with steering', () => {
-    let instructions = llmInstructions('fishaudio', { nonverbalSounds: {} });
+    let instructions = llmInstructions('fishaudio', { nonverbalSounds: false });
     expect(instructions).toBeDefined();
     expect(instructions).not.toContain('laughing');
     expect(instructions).not.toContain('clear throat');
     expect(instructions).toContain('Examples:');
 
-    instructions = llmInstructions('fishaudio', { nonverbalSounds: { laughing: true } });
+    instructions = llmInstructions('fishaudio', {
+      nonverbalSounds: { reflexSounds: false },
+    });
     expect(instructions).toBeDefined();
     expect(instructions).toContain('laughing');
     expect(instructions).not.toContain('clear throat');
@@ -212,6 +216,86 @@ describe('Fish Audio dialect', () => {
     });
   });
 
+  it('includes the register rule for every markup-capable provider', () => {
+    for (const provider of ['fishaudio', 'inworld', 'xai', 'cartesia']) {
+      expect(llmInstructions(provider), provider).toContain('REGISTER of the moment');
+    }
+  });
+
+  it('matches Fish register guidance to steering', () => {
+    const defaultInstructions = llmInstructions('fishaudio');
+    expect(defaultInstructions).toContain('Laughter belongs only');
+    expect(defaultInstructions).toContain('Save fillers for relaxed moments');
+
+    const composed = llmInstructions('fishaudio', {
+      nonverbalSounds: false,
+      disfluencies: false,
+    });
+    expect(composed?.toLowerCase()).not.toContain('laugh');
+    expect(composed?.toLowerCase()).not.toContain('filler');
+    expect(composed).not.toContain('Um, uh');
+  });
+
+  it('accepts a boolean for non-verbal sounds', () => {
+    const off = llmInstructions('fishaudio', { nonverbalSounds: false });
+    const on = llmInstructions('fishaudio', { nonverbalSounds: true });
+    const defaultInstructions = llmInstructions('fishaudio');
+    expect(off).not.toContain('type="sound"');
+    expect(off).not.toContain('laughing');
+    for (const instructions of [on, defaultInstructions]) {
+      expect(instructions).toContain('laughing, chuckling, clear throat');
+    }
+  });
+
+  it('renders all-on forms like omission', () => {
+    for (const provider of ['fishaudio', 'inworld', 'xai']) {
+      for (const nonverbalSounds of [true, {}]) {
+        const steering = { nonverbalSounds };
+        expect(steeringInstructions(provider, steering)).toBe('');
+        expect(llmInstructions(provider, steering)).toBe(llmInstructions(provider));
+      }
+    }
+    expect(steeringInstructions('fishaudio', { nonverbalSounds: false })).toBe('');
+    const partial = steeringInstructions('fishaudio', {
+      nonverbalSounds: { laughing: false },
+    });
+    expect(partial).toContain('clear-throat');
+    expect(partial.toLowerCase()).not.toContain('laugh');
+  });
+
+  it('treats a non-verbal object as a sparse opt-out', () => {
+    const steering = { nonverbalSounds: { laughing: false } };
+    const fish = llmInstructions('fishaudio', steering);
+    expect(fish).not.toContain('laughing');
+    expect(fish).toContain('clear throat');
+
+    const inworld = llmInstructions('inworld', steering);
+    expect(inworld).not.toContain('label="laugh"');
+    for (const kept of ['sigh', 'breathe', 'clear throat', 'cough', 'yawn']) {
+      expect(inworld).toContain(kept);
+    }
+
+    const xai = llmInstructions('xai', steering);
+    expect(xai).not.toContain('laugh-speak');
+    expect(xai).toContain('whisper');
+  });
+
+  it('merges steering sparsely over defaults', () => {
+    const resolved = resolveExpressiveOptions(
+      { speechSteering: {} },
+      { providerKey: 'fishaudio', defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS },
+    ).speechSteering!;
+    expect(resolved.disfluencies).toBe(true);
+    expect(resolved.nonverbalSounds).toBeUndefined();
+
+    const composed = resolveExpressiveOptions(
+      { speechSteering: { nonverbalSounds: false, disfluencies: false } },
+      { providerKey: 'fishaudio', defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS },
+    ).speechSteering!;
+    expect(composed.nonverbalSounds).toBe(false);
+    expect(composed.disfluencies).toBe(false);
+  });
+
   it('includes disfluent examples only when enabled', () => {
     const on = llmInstructions('fishaudio', { disfluencies: true });
     const off = llmInstructions('fishaudio', { disfluencies: false });

agents/src/tts/markup_utils.test.ts

@@ -2,9 +2,6 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { describe, expect, it } from 'vitest';
-import { isInstructions } from '../llm/chat_context.js';
-import { DEFAULT_EXPRESSIVE_OPTIONS } from '../voice/agent_session.js';
-import * as presets from '../voice/presets.js';
 import {
   TranscriptMarkupStripper,
   convertMarkup,
@@ -120,19 +117,6 @@ describe('xAI dialect', () => {
     expect(convertMarkup('xai', raw)).toBe('[laugh] [pause] [long-pause] <whisper>hi</whisper>');
     expect(normalizeMarkup('xai', raw)).toBe(raw);
   });
-
-  it('resolves presets to xAI-tuned bodies', () => {
-    for (const preset of [presets.CUSTOMER_SERVICE, presets.CASUAL]) {
-      const opts = presets.resolveOptions(preset, {
-        providerKey: 'xai',
-        defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS,
-      });
-      const tmpl = opts.ttsInstructionsTemplate!;
-      const body = isInstructions(tmpl) ? tmpl.common : tmpl;
-      // tuned body, not the agnostic default (which has no xai marker reference)
-      expect(body).toContain('<expr type="prosody" label="whisper">');
-    }
-  });
 });
 
 describe('normalizeMarkup', () => {

agents/src/voice/agent_activity.ts

@@ -97,6 +97,7 @@ import {
   DEFAULT_EXPRESSIVE_OPTIONS,
   type ExpressiveOptions,
   type TurnDetectionMode,
+  resolveExpressiveOptions,
 } from './agent_session.js';
 import {
   AudioRecognition,
@@ -137,7 +138,6 @@ import {
   updateInstructions,
 } from './generation.js';
 import type { PlaybackFinishedEvent, TimedString } from './io.js';
-import { resolveOptions as resolveExpressivePreset } from './presets.js';
 import { type InputDetails, SpeechHandle } from './speech_handle.js';
 import {
   ToolExecutor,
@@ -1923,18 +1923,16 @@ export class AgentActivity implements RecognitionHooks {
       expr = this.agentSession.sessionOptions.expressive;
     }
     if (typeof expr === 'object') {
-      // a `preset` selector resolves to the active TTS provider's tuned preset
-      // (falling back to the agnostic default); explicit fields override on top
       const providerKey = this.tts ? this.tts._markupProviderKey() : '';
-      return resolveExpressivePreset(expr, {
+      return resolveExpressiveOptions(expr, {
         providerKey,
         defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS,
       });
     }
     if (!expr) {
       return undefined;
     }
-    return resolveExpressivePreset(
+    return resolveExpressiveOptions(
       {},
       {
         providerKey: this.tts._markupProviderKey(),

agents/src/voice/agent_session.ts

@@ -27,7 +27,13 @@ import {
 import type { OverlappingSpeechEvent } from '../inference/interruption/types.js';
 import { getJobContext } from '../job.js';
 import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js';
-import { AgentHandoffItem, ChatContext, ChatMessage, Instructions } from '../llm/chat_context.js';
+import {
+  AgentHandoffItem,
+  ChatContext,
+  ChatMessage,
+  Instructions,
+  isInstructions,
+} from '../llm/chat_context.js';
 import type {
   LLM,
   RealtimeModel,
@@ -43,6 +49,7 @@ import { type ModelUsage, ModelUsageCollector, filterZeroValues } from '../metri
 import type { STT } from '../stt/index.js';
 import type { STTError } from '../stt/stt.js';
 import { traceTypes, tracer } from '../telemetry/index.js';
+import { steeringInstructions } from '../tts/_provider_format.js';
 import type { TTS, TTSError } from '../tts/tts.js';
 import {
   DEFAULT_API_CONNECT_OPTIONS,
@@ -92,7 +99,6 @@ import {
   type KeytermsOptions,
   resolveKeytermsOptions,
 } from './keyterm_detection.js';
-import type { Preset } from './presets.js';
 import { RecorderIO } from './recorder_io/index.js';
 import { RoomSessionTransport, SessionHost } from './remote_session.js';
 import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';
@@ -179,25 +185,20 @@ export function resolveRecordingOptions(
  * Configuration for the expressive pipeline.
  *
  * Controls how TTS markup instructions are injected into the LLM when expressive is
- * enabled. All keys are optional; common shapes:
- *
- * - `{ preset: Preset.CASUAL }` — a domain preset, resolved to the active
- *   TTS provider's tuned tags (see `voice/presets`). Prefer the `presets.*` constants.
- * - `{ preset: ..., ttsInstructionsAppend: '...' }` — a preset plus your own
- *   rules appended after it resolves.
- * - `{ ttsInstructionsTemplate: '...' }` — a fully custom prompt.
- *
- * Any explicit template overrides the corresponding part of the resolved preset; unset
- * parts fall back to the resolved preset (or the provider-agnostic default).
+ * enabled. The provider-neutral instructions infer the appropriate register from each
+ * conversational turn. Use `speechSteering` only to opt out of default behavior, or
+ * provide a custom template or appended instructions for application-specific rules.
  */
 export interface ExpressiveOptions {
-  preset?: Preset;
   speechSteering?: SpeechSteeringOptions;
   ttsInstructionsTemplate?: Instructions | string;
   ttsInstructionsAppend?: string;
 }
 
-/** Non-verbal vocalizations a TTS may produce. Omitted fields default to off. */
+/**
+ * Non-verbal vocalizations a TTS may produce. This is a sparse opt-out: omitted fields
+ * default to on, and a category set to `false` is never advertised to the LLM.
+ */
 export interface NonverbalOptions {
   laughing?: boolean;
   breathing?: boolean;
@@ -208,10 +209,17 @@ export interface NonverbalOptions {
   reflexSounds?: boolean;
 }
 
-/** Provider-agnostic controls for generated speech delivery. */
+/**
+ * Provider-agnostic controls for generated speech delivery. Every field is a sparse
+ * override on the default full sound vocabulary and light fillers.
+ */
 export interface SpeechSteeringOptions {
   disfluencies?: boolean;
-  nonverbalSounds?: NonverbalOptions;
+  /**
+   * `true` or omission keeps every sound, `false` disables every sound, and an object
+   * disables only categories explicitly set to `false`.
+   */
+  nonverbalSounds?: boolean | NonverbalOptions;
   pace?: 'slow' | 'normal' | 'fast';
 }
 
@@ -224,6 +232,37 @@ export const DEFAULT_EXPRESSIVE_OPTIONS: ExpressiveOptions = {
   speechSteering: { disfluencies: true },
 };
 
+function appendInstructions(template: Instructions | string, extra: string): Instructions {
+  if (isInstructions(template)) {
+    return new Instructions(template.common + '\n\n' + extra, {
+      audio: template.audio,
+      text: template.text,
+    });
+  }
+  return new Instructions(template + '\n\n' + extra);
+}
+
+/** Resolve expressive options against framework defaults for a provider. */
+export function resolveExpressiveOptions(
+  expr: ExpressiveOptions,
+  options: { providerKey: string; defaultOptions: ExpressiveOptions },
+): ExpressiveOptions {
+  const { providerKey, defaultOptions } = options;
+  let ttsTemplate = expr.ttsInstructionsTemplate ?? defaultOptions.ttsInstructionsTemplate!;
+  const speechSteering = {
+    ...defaultOptions.speechSteering,
+    ...expr.speechSteering,
+  };
+  const steering = steeringInstructions(providerKey, speechSteering);
+  if (steering) {
+    ttsTemplate = appendInstructions(ttsTemplate, steering);
+  }
+  if (expr.ttsInstructionsAppend) {
+    ttsTemplate = appendInstructions(ttsTemplate, expr.ttsInstructionsAppend);
+  }
+  return { ttsInstructionsTemplate: ttsTemplate, speechSteering };
+}
+
 export interface InternalSessionOptions<UserData> extends AgentSessionOptions<UserData> {
   turnHandling: InternalTurnHandlingOptions;
   useTtsAlignedTranscript: boolean;
@@ -376,7 +415,7 @@ export type AgentSessionOptions<UserData = UnknownUserData> = {
    * provider's markup tags (emotion/expression/pauses/...), the tags are converted
    * to the provider's native syntax before synthesis, and stripped from the room
    * transcript. Pass `true` for the provider-agnostic default prompt, or an
-   * {@link ExpressiveOptions} (e.g. a `presets.*` constant) to tune it.
+   * {@link ExpressiveOptions} to tune it.
    * @defaultValue false
    */
   expressive?: boolean | ExpressiveOptions;

agents/src/voice/index.ts

@@ -52,7 +52,6 @@ export {
   createTimedString,
   isTimedString,
 } from './io.js';
-export * as presets from './presets.js';
 export * from './report.js';
 export * from './room_io/index.js';
 export { RunContext } from './run_context.js';