Merge pull request #52 from OHF-Voice/synesthesiam-20260514-intent-handle

Extend intent recognition and handling

Author: synesthesiam

CHANGELOG.md

@@ -1,7 +1,10 @@
 # Changelog
 
-## Unreleased
+## 1.9.0
 
+- Add `intents-start` and `intents-end` to allow for multiple intents to be recognized
+- Add `supports_home_control` flag to indicate an intent handling program may execute home control actions
+- Add `user-event` for user-defined events
 - Add `wake_words` query parameter to wake server
 
 ## 1.8.0

README.md

@@ -124,6 +124,7 @@ Describe available services.
             * `description` - human-readable description (string, optional)
             * `version` - version of the model (string, optional)
         * `supports_handled_streaming` - true if program can stream response chunks
+        * `supports_home_control` - true if program may execute home control actions during handling
     * `intent` - list intent recognition services (optional)
         * `models` - list of available models (required)
             * `name` - unique name (required)
@@ -236,6 +237,14 @@ Recognizes intents from text.
 * `not-recognized` - response indicating no intent was recognized
     * `text` - response for user (string, optional)
     * `context` - context for next interactions (object, optional)
+    
+Multiple intents:
+
+1. `intents-start` - signals one or more intents will follow
+    * `context` - context from previous interactions (object, optional)
+2. `intent` - one or more intents sent as normal
+    * Older clients will only process the first intent
+3. `intents-stop` - end of intents
 
 ### Intent Handling
 
@@ -311,6 +320,13 @@ Pipelines are run on the server, but can be triggered remotely from the server a
 * `timer-finished` - timer finished without being cancelled
     * `id` - unique id of timer (string, required)
 
+### Miscellaneous
+
+* `user-event` - user-defined event
+    * `name` - name of the user event type (string, required)
+    * `data` - data for user event (object, optional)
+    * `context` - context from previous interactions (object, optional)
+
 ## Event Flow
 
 * → is an event from client to server
@@ -400,6 +416,15 @@ Streaming:
 2. ← `intent` if successful
 3. ← `not-recognized` if not successful
 
+For multiple intents:
+
+1. → `recognize` (required)
+2. → `intents-start` if successful
+2. → `intent` if successful
+    * One or more intents
+2. → `intents-end` if successful
+3. ← `not-recognized` if not successful
+
 ### Intent Handling
 
 For structured intents:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "wyoming"
-version = "1.8.0"
+version = "1.9.0"
 description = "Peer-to-peer protocol for voice assistants"
 readme = "README.md"
 requires-python = ">=3.8"

tests/test_eventable.py

@@ -135,6 +135,7 @@ def all_unique_eventables() -> List[Type[Eventable]]:
                     )
                 ],
                 supports_handled_streaming=True,
+                supports_home_control=True,
             )
         ],
         "intent": [
@@ -250,6 +251,8 @@ def all_unique_eventables() -> List[Type[Eventable]]:
         "context": TEST_CONTEXT,
     },
     "NotRecognized": {"text": TEST_TEXT, "context": TEST_CONTEXT},
+    "IntentsStart": {"context": TEST_CONTEXT},
+    "IntentsStop": {"context": TEST_CONTEXT},
     # handle
     "Handled": {
         "text": TEST_TEXT,
@@ -287,6 +290,11 @@ def all_unique_eventables() -> List[Type[Eventable]]:
     "Ping": {},
     "Pong": {},
     "RunPipeline": {"start_stage": PipelineStage.ASR, "end_stage": PipelineStage.TTS},
+    "UserEvent": {
+        "name": TEST_NAME,
+        "data": {TEST_ID: TEST_TEXT},
+        "context": TEST_CONTEXT,
+    },
 }
 
 

wyoming/http/intent_server.py

@@ -2,15 +2,15 @@
 
 import logging
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, List, Union
 
 from flask import request
 
 from wyoming.asr import Transcript
 from wyoming.client import AsyncClient
 from wyoming.error import Error
 from wyoming.handle import Handled, NotHandled
-from wyoming.intent import Intent, NotRecognized
+from wyoming.intent import Intent, IntentsStart, IntentsStop, NotRecognized
 
 from .shared import get_app, get_argument_parser
 
@@ -27,7 +27,7 @@ def main():
     app = get_app("intent", CONF_PATH, args)
 
     @app.route("/api/recognize-intent", methods=["POST", "GET"])
-    async def api_stt() -> Dict[str, Any]:
+    async def api_recognize_intent() -> Union[Dict[str, Any], List[Dict[str, Any]]]:
         uri = request.args.get("uri", args.uri)
         if not uri:
             raise ValueError("URI is required")
@@ -45,42 +45,60 @@ async def api_stt() -> Dict[str, Any]:
         async with AsyncClient.from_uri(uri) as client:
             await client.write_event(Transcript(text=text, language=language).event())
 
+            type_name = "unknown"
+            results: List[Dict[str, Any]] = []
+            is_intent_list = False
+
             while True:
                 event = await client.read_event()
                 if event is None:
                     raise RuntimeError("Client disconnected")
 
-                success = False
-                type_name = "unknown"
-                result: Dict[str, Any] = {}
+                if IntentsStart.is_type(event.type):
+                    is_intent_list = True
+                    continue
 
                 if Intent.is_type(event.type):
-                    success = True
                     type_name = "intent"
                     intent = Intent.from_event(event)
-                    result = intent.to_dict()
-                elif Handled.is_type(event.type):
-                    success = True
+                    results.append(intent.to_dict())
+                    if not is_intent_list:
+                        break
+
+                if IntentsStop.is_type(event.type):
+                    break
+
+                if Handled.is_type(event.type):
                     type_name = "handled"
                     handled = Handled.from_event(event)
-                    result = handled.to_dict()
-                elif NotRecognized.is_type(event.type):
-                    success = False
+                    results.append(handled.to_dict())
+                    break
+
+                if NotRecognized.is_type(event.type):
                     type_name = "not-recognized"
                     not_recognized = NotRecognized.from_event(event)
-                    result = not_recognized.to_dict()
-                elif NotHandled.is_type(event.type):
-                    success = False
+                    results.append(not_recognized.to_dict())
+                    break
+
+                if NotHandled.is_type(event.type):
                     type_name = "not-handled"
                     not_handled = NotHandled.from_event(event)
-                    result = not_handled.to_dict()
-                elif Error.is_type(event.type):
+                    results.append(not_handled.to_dict())
+                    break
+
+                if Error.is_type(event.type):
                     error = Error.from_event(event)
                     raise RuntimeError(
                         f"Unexpected error from client: code={error.code}, text={error.text}"
                     )
 
-                return {"success": success, "type": type_name, "result": result}
+            if len(results) == 0:
+                return {"success": False, "type": "unknown", "result": {}}
+
+            if len(results) == 1:
+                return {"success": True, "type": type_name, "result": results[0]}
+
+            return {"success": True, "type": type_name, "result": results}
 
     app.run(args.host, args.port)
 

wyoming/info.py

@@ -135,6 +135,9 @@ class HandleProgram(Artifact):
     supports_handled_streaming: bool = False
     """True if handled response streaming events are supported."""
 
+    supports_home_control: bool = False
+    """True if the service may execute home control actions during handling."""
+
 
 # -----------------------------------------------------------------------------
 

wyoming/intent.py

@@ -9,6 +9,8 @@
 _RECOGNIZE_TYPE = "recognize"
 _INTENT_TYPE = "intent"
 _NOT_RECOGNIZED_TYPE = "not-recognized"
+_INTENTS_START_TYPE = "intents-start"
+_INTENTS_STOP_TYPE = "intents-stop"
 
 
 @dataclass
@@ -137,3 +139,69 @@ def from_event(event: Event) -> "NotRecognized":
         return NotRecognized(
             text=event.data.get("text"), context=event.data.get("context")
         )
+
+
+@dataclass
+class IntentsStart(Eventable):
+    """Start of one or more intents.
+
+    Event flow:
+    intents-start
+    intent
+    [intent]...
+    intents-stop
+    """
+
+    context: Optional[Dict[str, Any]] = None
+    """Context for next interaction."""
+
+    @staticmethod
+    def is_type(event_type: str) -> bool:
+        return event_type == _INTENTS_START_TYPE
+
+    def event(self) -> Event:
+        data: Dict[str, Any] = {}
+        if self.context is not None:
+            data["context"] = self.context
+
+        return Event(type=_INTENTS_START_TYPE, data=data)
+
+    @staticmethod
+    def from_event(event: Event) -> "IntentsStart":
+        if not event.data:
+            return IntentsStart()
+
+        return IntentsStart(context=event.data.get("context"))
+
+
+@dataclass
+class IntentsStop(Eventable):
+    """End of intent series.
+
+    Event flow:
+    intents-start
+    intent
+    [intent]...
+    intents-stop
+    """
+
+    context: Optional[Dict[str, Any]] = None
+    """Context for next interaction."""
+
+    @staticmethod
+    def is_type(event_type: str) -> bool:
+        return event_type == _INTENTS_STOP_TYPE
+
+    def event(self) -> Event:
+        data: Dict[str, Any] = {}
+        if self.context is not None:
+            data["context"] = self.context
+
+        return Event(type=_INTENTS_STOP_TYPE, data=data)
+
+    @staticmethod
+    def from_event(event: Event) -> "IntentsStop":
+        if not event.data:
+            return IntentsStop()
+
+        return IntentsStop(context=event.data.get("context"))

wyoming/user_event.py

@@ -0,0 +1,42 @@
+"""User-defined events."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from .event import Event, Eventable
+
+_USER_EVENT_TYPE = "user-event"
+
+
+@dataclass
+class UserEvent(Eventable):
+    """User-defined event."""
+
+    name: str
+    """Name of user event type."""
+
+    data: Optional[Dict[str, Any]] = None
+    """User event data."""
+
+    context: Optional[Dict[str, Any]] = None
+    """Context from previous interactions."""
+
+    @staticmethod
+    def is_type(event_type: str) -> bool:
+        return event_type == _USER_EVENT_TYPE
+
+    def event(self) -> Event:
+        data: Dict[str, Any] = {"name": self.name}
+        if self.data is not None:
+            data["data"] = self.data
+        if self.context is not None:
+            data["context"] = self.context
+        return Event(type=_USER_EVENT_TYPE, data=data)
+
+    @staticmethod
+    def from_event(event: Event) -> "UserEvent":
+        return UserEvent(
+            name=event.data["name"],
+            data=event.data.get("data"),
+            context=event.data.get("context"),
+        )