From 59f83267f0d0263d4e7428a935b41042b9faa06e Mon Sep 17 00:00:00 2001 From: Brett Kinny Date: Sat, 11 Jul 2026 22:03:42 +1000 Subject: [PATCH 1/2] fix UAT release blockers --- bridge.py | 16 +- custom-providers/pi_voice/README.md | 24 +- custom-providers/pi_voice/pi_client.py | 17 +- custom-providers/pi_voice/pi_voice.py | 116 +++++++- .../pi_voice/tests/test_pi_client.py | 32 ++ .../pi_voice/tests/test_pi_voice.py | 140 ++++++++- .../xiaozhi-patches/http_server.py | 4 + .../textMessageHandlerRegistry.py | 1 + docs/architecture.md | 8 +- docs/brain.md | 28 +- docs/cookbook/change-persona.md | 16 +- docs/emoji-mapping.md | 29 +- docs/faq.md | 8 +- docs/kid-mode.md | 100 +++---- docs/llm-backends.md | 7 +- docs/modes.md | 4 +- docs/protocols.md | 2 +- docs/quickstart.md | 8 +- docs/troubleshooting.md | 17 +- docs/uat-runbook.md | 10 +- docs/voice-pipeline.md | 2 +- dotty-behaviour/config.py | 4 +- dotty-behaviour/tests/test_config.py | 38 +++ dotty-behaviour/tests/test_routes_vision.py | 15 + firmware | 2 +- receiveAudioHandle.py | 98 +++++- scripts/uat-slice.py | 6 +- tests/test_asr_name_corrections.py | 101 +++++++ tests/test_bridge_routes.py | 22 ++ tests/test_dance_state_lifecycle.py | 281 ++++++++++++++++++ tests/test_device_command.py | 2 + tests/test_uat_slice.py | 90 ++++++ tests/test_voice_content_filter.py | 48 +-- 33 files changed, 1093 insertions(+), 203 deletions(-) create mode 100644 dotty-behaviour/tests/test_config.py create mode 100644 tests/test_asr_name_corrections.py create mode 100644 tests/test_dance_state_lifecycle.py create mode 100644 tests/test_uat_slice.py diff --git a/bridge.py b/bridge.py index 974dbd0..b4783d2 100644 --- a/bridge.py +++ b/bridge.py @@ -575,9 +575,19 @@ def _dashboard_perception_recent_getter( def _dashboard_state_getter() -> str: - """Return the current State of Dotty. Without a perception bus the - bridge can't know โ€” fall through to 'idle' so the dashboard renders - safely. Real state is mirrored on dotty-behaviour.""" + """Return the current State mirrored by dotty-behaviour. + + The deployment has one robot, so use the first valid per-device state. + If behaviour is unavailable or has not observed a state transition yet, + retain the dashboard's historical safe fallback of ``idle``. + """ + states = _dashboard_perception_state_getter() + for device_state in states.values(): + if not isinstance(device_state, dict): + continue + current = device_state.get("current_state") + if isinstance(current, str) and current: + return current return "idle" diff --git a/custom-providers/pi_voice/README.md b/custom-providers/pi_voice/README.md index 7d9f697..e261fab 100644 --- a/custom-providers/pi_voice/README.md +++ b/custom-providers/pi_voice/README.md @@ -147,16 +147,20 @@ LLM providers. pi's `docs/rpc.md`; `assistantMessageEvent` filtering rule from the spike telemetry. -## Open questions still on the table - -- **Memory write-back.** PiVoiceLLM does not yet persist conversation - turns or remember-markers. Memory write-back belongs in the pi - extension: a small write (sqlite_brain_db.write) triggered by a - `[REMEMBER: โ€ฆ]` marker in the final assistant text, plus a per-turn - log row. -- **Persona file location.** The pi extension reads from - `/mnt/user/appdata/dotty-pi/persona/` (bind-mounted into the container). - Wiring is stable; runtime persona-swap mechanism TBD. +## Runtime prompt policy and persona state + +The production Pi command uses `--no-context-files`, so files under the +bind-mounted `/root/.pi/persona/` tree are intentionally not loaded into voice +turns. They are live operator state and `deploy-dotty-pi.sh` does not overwrite +them. Voice-critical routing and output policy must therefore live in the +versioned `PiVoiceLLM` per-turn prompt (`pi_voice.py` and `textUtils.py`), which +is shipped by `deploy-bridge-unraid.sh`. Do not rely on a repo persona edit for +a production voice fix unless a separate, explicit persona-management workflow +is introduced. + +Conversation logging and explicit memory writes are implemented by the +`dotty-pi-ext` `agent_end` listener and `remember` tools respectively; the old +`[REMEMBER: โ€ฆ]` marker protocol is not part of the live PiVoiceLLM path. ## See also diff --git a/custom-providers/pi_voice/pi_client.py b/custom-providers/pi_voice/pi_client.py index d54c4a4..7431bf3 100644 --- a/custom-providers/pi_voice/pi_client.py +++ b/custom-providers/pi_voice/pi_client.py @@ -247,10 +247,19 @@ def iter_turn_text(self, prompt: str) -> Iterator[str]: if ftype == "message_update": ame = frame.get("assistantMessageEvent") - if isinstance(ame, dict) and ame.get("type") == "text_delta": - delta = ame.get("delta") - if isinstance(delta, str) and delta: - yield delta + if isinstance(ame, dict): + if ame.get("type") == "text_delta": + delta = ame.get("delta") + if isinstance(delta, str) and delta: + yield delta + elif ame.get("type") == "toolcall_end": + tool_call = ame.get("toolCall") + if isinstance(tool_call, dict): + logger.info( + "PiClient: tool call name=%s id=%s", + tool_call.get("name", "unknown"), + tool_call.get("id", "unknown"), + ) # thinking_delta, thinking_start, thinking_end and any # other message_update sub-types are filtered out here. continue diff --git a/custom-providers/pi_voice/pi_voice.py b/custom-providers/pi_voice/pi_voice.py index 6b991b5..39036be 100644 --- a/custom-providers/pi_voice/pi_voice.py +++ b/custom-providers/pi_voice/pi_voice.py @@ -33,7 +33,10 @@ from __future__ import annotations +import json import os +import unicodedata +from pathlib import Path from typing import Iterator from .pi_client import PiClient, PiClientError, make_default_pi_client @@ -61,6 +64,8 @@ def setup_logging(): # type: ignore[no-redef] # by absolute path. Both code paths end up with the same module. try: from core.utils.textUtils import ( # type: ignore + ALLOWED_EMOJIS, + FALLBACK_EMOJI, build_turn_suffix, filter_tts_stream, ) @@ -73,6 +78,8 @@ def setup_logging(): # type: ignore[no-redef] assert _spec is not None and _spec.loader is not None _tu = _ilu.module_from_spec(_spec) _spec.loader.exec_module(_tu) + ALLOWED_EMOJIS = _tu.ALLOWED_EMOJIS # type: ignore[attr-defined] + FALLBACK_EMOJI = _tu.FALLBACK_EMOJI # type: ignore[attr-defined] build_turn_suffix = _tu.build_turn_suffix # type: ignore[attr-defined] filter_tts_stream = _tu.filter_tts_stream # type: ignore[attr-defined] @@ -82,6 +89,18 @@ def setup_logging(): # type: ignore[no-redef] def _read_kid_mode() -> bool: + """Read the shared runtime toggle, falling back to startup config.""" + state_file = Path(os.environ.get( + "DOTTY_KID_MODE_STATE", "/var/lib/dotty-bridge/state/kid-mode", + )) + try: + value = state_file.read_text().strip().lower() + if value in ("true", "1", "yes"): + return True + if value in ("false", "0", "no"): + return False + except OSError: + pass return os.environ.get("DOTTY_KID_MODE", "true").lower() in ("1", "true", "yes") @@ -91,10 +110,44 @@ def _last_user_text(dialogue: list[dict]) -> str: entry is the utterance we want pi to react to.""" for msg in reversed(dialogue): if msg.get("role") == "user": - return str(msg.get("content") or "") + return _normalise_user_content(msg.get("content")) return "" +def _normalise_user_content(content: object) -> str: + """Extract xiaozhi's user text without leaking its JSON envelope to pi. + + Depending on where the dialogue was assembled, ``content`` may already be + plain text, a ``{"content": "..."}`` mapping, or the JSON encoding of that + mapping. Unknown JSON is kept verbatim: silently discarding or reshaping a + genuine user utterance would be worse than passing it through. + """ + if isinstance(content, dict): + inner = content.get("content") + return inner if isinstance(inner, str) else str(content or "") + if not isinstance(content, str): + return str(content or "") + stripped = content.strip() + if stripped.startswith("{") and stripped.endswith("}"): + try: + decoded = json.loads(stripped) + except json.JSONDecodeError: + return content + if isinstance(decoded, dict) and isinstance(decoded.get("content"), str): + return decoded["content"] + return content + + +_VOICE_TOOL_ROUTING = ( + "\n\nVOICE TOOL ROUTING: Decide whether to use a registered tool before " + "composing the spoken reply. Use the matching tool when the user asks you " + "to remember or recall, play a song, look or take a photo, or solve a " + "question that needs careful reasoning. Call it first and use its result. " + "Never claim an action succeeded unless its tool succeeded. The reply " + "constraints below apply only to final spoken text, not to tool calls." +) + + def _wrap_with_sandwich(user_text: str, kid_mode: bool) -> str: """Append the HARD CONSTRAINTS suffix to the user's text via the shared textUtils.build_turn_suffix contract โ€” emoji-prefix @@ -102,7 +155,50 @@ def _wrap_with_sandwich(user_text: str, kid_mode: bool) -> str: this Dotty drifts into Chinese, multi-paragraph replies, and (in kid_mode) unsafe topics, since qwen3.5:4b's base behaviour doesn't encode any of those constraints.""" - return user_text + build_turn_suffix(kid_mode) + return user_text + _VOICE_TOOL_ROUTING + build_turn_suffix(kid_mode) + + +def _enforce_leading_emoji(chunks: Iterator[str]) -> Iterator[str]: + """Guarantee the firmware's leading-glyph face contract. + + Pi is prompted to start with an allowed emoji, but model compliance is not + an output guarantee. Buffer only leading whitespace, then either pass an + allowed emoji through or replace a missing/disallowed leading glyph with + the neutral fallback before the model text. + """ + leading: list[str] = [] + saw_content = False + for chunk in chunks: + if not chunk: + continue + if not saw_content: + leading.append(chunk) + so_far = "".join(leading).lstrip() + if not so_far: + continue + saw_content = True + if not any(so_far.startswith(emoji) for emoji in ALLOWED_EMOJIS): + yield f"{FALLBACK_EMOJI} " + # Do not leave a disallowed model emoji after the fallback: + # `๐Ÿ˜ โค๏ธ hello` still violates the exactly-one-face contract. + # Consume the leading symbol plus emoji presentation/joiner + # codepoints, while leaving ordinary punctuation and text. + if so_far and unicodedata.category(so_far[0]) == "So": + end = 1 + while end < len(so_far) and ( + so_far[end] in ("\ufe0f", "\u200d") + or 0x1F3FB <= ord(so_far[end]) <= 0x1F3FF + or unicodedata.category(so_far[end]) == "So" + ): + end += 1 + so_far = so_far[end:].lstrip() + if so_far: + yield so_far + continue + yield chunk + + if not saw_content: + yield f"{FALLBACK_EMOJI} (no response)" class LLMProvider(LLMProviderBase): @@ -112,9 +208,8 @@ def __init__(self, config: dict, *, client: PiClient | None = None): self._container = config.get("container_name") or os.environ.get( "DOTTY_PI_CONTAINER", "dotty-pi", ) - # KID_MODE is a process-start snapshot. Toggling kid_mode requires - # a container restart, which already happens via the bridge's - # existing restart path. + # Initial value is logged for diagnostics. response() refreshes this + # from the bridge/xiaozhi shared state file on every turn. self._kid_mode = _read_kid_mode() # `client` is injected by tests; production passes None to get # the env-configured default. @@ -129,9 +224,10 @@ def __init__(self, config: dict, *, client: PiClient | None = None): # xiaozhi-server's voice loop calls this as a sync generator. # Each yielded string becomes a TTS chunk. def response(self, session_id, dialogue, **kwargs) -> Iterator[str]: + self._kid_mode = _read_kid_mode() user_text = _last_user_text(dialogue) if not user_text: - yield "(empty turn)" + yield f"{FALLBACK_EMOJI} (empty turn)" return prompt = _wrap_with_sandwich(user_text, self._kid_mode) @@ -148,9 +244,11 @@ def response(self, session_id, dialogue, **kwargs) -> Iterator[str]: # #157: kid-mode blocked-content filter on TTS-bound output. # Full-turn buffered โ€” the filter drains the pi RPC stream through # agent_end before making an atomic allow/replace decision. - # kid_mode off is a transparent passthrough. + # Emoji enforcement precedes filtering, matching OpenAICompat: in + # kid mode the filter still makes one atomic whole-turn decision; + # outside it, chunks stream after the first meaningful one. for chunk in filter_tts_stream( - self._client.iter_turn_text(prompt), + _enforce_leading_emoji(self._client.iter_turn_text(prompt)), self._kid_mode, on_hit=self._on_filter_hit, ): @@ -159,7 +257,7 @@ def response(self, session_id, dialogue, **kwargs) -> Iterator[str]: logger.error("PiVoiceLLM turn failed: %s", exc) for line in self._client.recent_stderr()[-5:]: logger.error(" pi.stderr: %s", line) - yield "(brain offline โ€” try again in a moment)" + yield f"{FALLBACK_EMOJI} (brain offline โ€” try again in a moment)" def _on_filter_hit(self, tier: str, match) -> None: # Local logging only โ€” the Prometheus counter / safety ring live in diff --git a/custom-providers/pi_voice/tests/test_pi_client.py b/custom-providers/pi_voice/tests/test_pi_client.py index 1b2fcba..50e2b5e 100644 --- a/custom-providers/pi_voice/tests/test_pi_client.py +++ b/custom-providers/pi_voice/tests/test_pi_client.py @@ -212,6 +212,38 @@ def feed(): client.close() +class TestToolCallTelemetry(unittest.TestCase): + def test_completed_tool_call_is_logged_at_info_without_arguments(self): + fake = FakePopen() + client = make_client(fake) + try: + def feed(): + fake.emit({ + "id": "turn-1", "type": "response", + "command": "prompt", "success": True, + }) + fake.emit({ + "type": "message_update", + "assistantMessageEvent": { + "type": "toolcall_end", + "toolCall": { + "type": "toolCall", "id": "tool-7", + "name": "remember", "arguments": {"fact": "private"}, + }, + }, + }) + fake.emit({"type": "agent_end"}) + + threading.Thread(target=feed, daemon=True).start() + with self.assertLogs("pi_client", level="INFO") as logs: + self.assertEqual(list(client.iter_turn_text("remember this")), []) + joined = "\n".join(logs.output) + self.assertIn("tool call name=remember id=tool-7", joined) + self.assertNotIn("private", joined) + finally: + client.close() + + class TestUiAutoCancel(unittest.TestCase): def test_dialog_methods_get_auto_cancelled(self): fake = FakePopen() diff --git a/custom-providers/pi_voice/tests/test_pi_voice.py b/custom-providers/pi_voice/tests/test_pi_voice.py index e9e82fa..7725fa7 100644 --- a/custom-providers/pi_voice/tests/test_pi_voice.py +++ b/custom-providers/pi_voice/tests/test_pi_voice.py @@ -9,8 +9,11 @@ import os import sys +import tempfile import unittest +from pathlib import Path from typing import Iterator +from unittest.mock import patch HERE = os.path.dirname(os.path.abspath(__file__)) PROVIDER_DIR = os.path.dirname(HERE) @@ -23,7 +26,12 @@ # pi_voice catches pi_voice.pi_client.PiClientError, and `from pi_client # import PiClientError` would give us a *different* class object even # though the source is identical, so isinstance/except wouldn't match. -from pi_voice import LLMProvider, PiClientError, _wrap_with_sandwich # noqa: E402 +from pi_voice import ( # noqa: E402 + LLMProvider, + PiClientError, + _wrap_with_sandwich, +) +from pi_voice.pi_voice import _last_user_text # noqa: E402 class FakeClient: @@ -68,8 +76,8 @@ def test_suffix_appended_kid_mode_on(self): provider = LLMProvider({}, client=client) # type: ignore[arg-type] list(provider.response("sess-1", [{"role": "user", "content": "Hello"}])) self.assertEqual(len(client.prompts), 1) - expected = "Hello" + textUtils.build_turn_suffix(True) - self.assertEqual(client.prompts[0], expected) + self.assertTrue(client.prompts[0].startswith("Hello\n\nVOICE TOOL ROUTING:")) + self.assertTrue(client.prompts[0].endswith(textUtils.build_turn_suffix(True))) # Sanity: the kid-mode-specific bullets must be in the suffix. self.assertIn("YOUNG CHILD", client.prompts[0]) self.assertIn("SELF-HARM EXCEPTION", client.prompts[0]) @@ -80,20 +88,87 @@ def test_suffix_appended_kid_mode_off(self): client.script_turn(["๐Ÿ˜ OK"]) provider = LLMProvider({}, client=client) # type: ignore[arg-type] list(provider.response("sess-1", [{"role": "user", "content": "Hi"}])) - expected = "Hi" + textUtils.build_turn_suffix(False) - self.assertEqual(client.prompts[0], expected) + self.assertTrue(client.prompts[0].startswith("Hi\n\nVOICE TOOL ROUTING:")) + self.assertTrue(client.prompts[0].endswith(textUtils.build_turn_suffix(False))) # Adult mode: still has emoji-prefix / English-only / no-Markdown # bullets, but NOT the kid-specific ones. self.assertIn("EXACTLY ONE emoji", client.prompts[0]) self.assertNotIn("YOUNG CHILD", client.prompts[0]) def test_wrap_helper_pure(self): - # build_turn_suffix is the source of truth โ€” _wrap_with_sandwich - # is just `user + suffix`. This pins that contract so a future - # refactor can't quietly move pre/postfix logic around. + # Tool routing must precede the final spoken-output constraints. wrapped = _wrap_with_sandwich("hi", True) self.assertTrue(wrapped.startswith("hi")) - self.assertEqual(wrapped, "hi" + textUtils.build_turn_suffix(True)) + self.assertLess(wrapped.index("VOICE TOOL ROUTING"), wrapped.index("HARD CONSTRAINTS")) + self.assertIn("not to tool calls", wrapped) + + def test_json_wrapped_user_content_is_unwrapped(self): + dialogue = [{"role": "user", "content": '{"content": "remember purple"}'}] + self.assertEqual(_last_user_text(dialogue), "remember purple") + + def test_mapping_wrapped_user_content_is_unwrapped(self): + dialogue = [{"role": "user", "content": {"content": "think carefully"}}] + self.assertEqual(_last_user_text(dialogue), "think carefully") + + def test_unknown_or_invalid_json_text_is_preserved(self): + for content in ('{"question": "why"}', "{not json}"): + self.assertEqual( + _last_user_text([{"role": "user", "content": content}]), + content, + ) + + def test_shared_state_file_refreshes_kid_mode_each_turn(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "kid-mode" + state_file.write_text("true") + old_path = os.environ.get("DOTTY_KID_MODE_STATE") + os.environ["DOTTY_KID_MODE_STATE"] = str(state_file) + self.addCleanup( + lambda: ( + os.environ.__setitem__("DOTTY_KID_MODE_STATE", old_path) + if old_path is not None + else os.environ.pop("DOTTY_KID_MODE_STATE", None) + ) + ) + + client = FakeClient() + client.script_turn(["๐Ÿ˜Š first"]) + client.script_turn(["๐Ÿ˜ second"]) + provider = LLMProvider({}, client=client) # type: ignore[arg-type] + list(provider.response("s", [{"role": "user", "content": "one"}])) + + state_file.write_text("false") + list(provider.response("s", [{"role": "user", "content": "two"}])) + + self.assertIn("YOUNG CHILD", client.prompts[0]) + self.assertNotIn("YOUNG CHILD", client.prompts[1]) + + def test_malformed_shared_state_falls_back_to_environment(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "kid-mode" + state_file.write_text("not-a-boolean") + old_path = os.environ.get("DOTTY_KID_MODE_STATE") + old_mode = os.environ.get("DOTTY_KID_MODE") + os.environ["DOTTY_KID_MODE_STATE"] = str(state_file) + os.environ["DOTTY_KID_MODE"] = "false" + + def restore_env() -> None: + for name, value in ( + ("DOTTY_KID_MODE_STATE", old_path), + ("DOTTY_KID_MODE", old_mode), + ): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + self.addCleanup(restore_env) + client = FakeClient() + client.script_turn(["๐Ÿ˜ adult mode"]) + provider = LLMProvider({}, client=client) # type: ignore[arg-type] + list(provider.response("s", [{"role": "user", "content": "hello"}])) + + self.assertNotIn("YOUNG CHILD", client.prompts[0]) class TestEmptyTurn(unittest.TestCase): @@ -102,7 +177,7 @@ def test_no_user_message_short_circuits(self): client = FakeClient() provider = LLMProvider({}, client=client) # type: ignore[arg-type] out = list(provider.response("sess-1", [{"role": "system", "content": "..."}])) - self.assertEqual(out, ["(empty turn)"]) + self.assertEqual(out, [f"{textUtils.FALLBACK_EMOJI} (empty turn)"]) self.assertEqual(client.prompts, [], "PiClient must not be called for empty dialogue") @@ -126,7 +201,50 @@ def test_client_error_yields_fallback(self): client.script_turn([], error=PiClientError("pi crashed")) provider = LLMProvider({}, client=client) # type: ignore[arg-type] out = list(provider.response("s", [{"role": "user", "content": "anything"}])) - self.assertEqual(out, ["(brain offline โ€” try again in a moment)"]) + self.assertEqual(out, [f"{textUtils.FALLBACK_EMOJI} (brain offline โ€” try again in a moment)"]) + + +class TestLeadingEmojiContract(unittest.TestCase): + def _response(self, chunks: list[str], *, kid_mode: bool = False) -> list[str]: + env = { + "DOTTY_KID_MODE": "true" if kid_mode else "false", + "DOTTY_KID_MODE_STATE": "/nonexistent/dotty-test-kid-mode", + } + with patch.dict(os.environ, env): + client = FakeClient() + client.script_turn(chunks) + provider = LLMProvider({}, client=client) # type: ignore[arg-type] + return list(provider.response("s", [{"role": "user", "content": "hello"}])) + + def test_missing_emoji_gets_fallback_before_first_text(self): + out = self._response(["Hello", " there"]) + self.assertEqual(out[0], f"{textUtils.FALLBACK_EMOJI} ") + self.assertEqual("".join(out), f"{textUtils.FALLBACK_EMOJI} Hello there") + + def test_leading_whitespace_never_precedes_emoji(self): + out = self._response([" ", "Hello"]) + self.assertTrue(out[0].startswith(textUtils.FALLBACK_EMOJI)) + self.assertEqual("".join(out), f"{textUtils.FALLBACK_EMOJI} Hello") + + def test_allowed_emoji_is_not_double_prefixed(self): + out = self._response(["๐Ÿ˜Š Hello"]) + self.assertEqual(out, ["๐Ÿ˜Š Hello"]) + + def test_disallowed_leading_emoji_is_replaced_not_retained(self): + out = self._response(["โค๏ธ Hello"]) + self.assertEqual("".join(out), f"{textUtils.FALLBACK_EMOJI} Hello") + + def test_disallowed_single_codepoint_emoji_is_replaced(self): + out = self._response(["๐Ÿ˜‚ Hello"]) + self.assertEqual("".join(out), f"{textUtils.FALLBACK_EMOJI} Hello") + + def test_empty_model_stream_gets_emoji_fallback(self): + out = self._response([]) + self.assertEqual(out, [f"{textUtils.FALLBACK_EMOJI} (no response)"]) + + def test_kid_filter_still_replaces_the_complete_turn(self): + out = self._response(["Hello ", "cocaine"], kid_mode=True) + self.assertEqual(out, [textUtils.CONTENT_FILTER_REPLACEMENT]) if __name__ == "__main__": diff --git a/custom-providers/xiaozhi-patches/http_server.py b/custom-providers/xiaozhi-patches/http_server.py index 0290f20..b71585e 100644 --- a/custom-providers/xiaozhi-patches/http_server.py +++ b/custom-providers/xiaozhi-patches/http_server.py @@ -230,6 +230,10 @@ async def _dotty_set_state(self, request: "web.Request") -> "web.Response": conn, err = _dotty_resolve_conn(device_id) if err is not None: return err + # Publish intent before spawning the MCP send. A finishing dance uses + # this synchronous value to avoid restoring IDLE over an admin state + # change while the firmware state_changed echo is still in flight. + conn._dotty_desired_state = state _spawn( _dotty_device_command.call_tool( conn, "self.robot.set_state", {"state": state}, diff --git a/custom-providers/xiaozhi-patches/textMessageHandlerRegistry.py b/custom-providers/xiaozhi-patches/textMessageHandlerRegistry.py index 4394444..b596545 100644 --- a/custom-providers/xiaozhi-patches/textMessageHandlerRegistry.py +++ b/custom-providers/xiaozhi-patches/textMessageHandlerRegistry.py @@ -111,6 +111,7 @@ async def handle(self, conn, msg_json: Dict[str, Any]) -> None: new_state = ((msg_json.get("data") or {}).get("state") or "").strip().lower() if new_state: conn.current_state = new_state + conn._dotty_desired_state = new_state except Exception: pass # Listen on `face_detected`. Prior firmware also emitted diff --git a/docs/architecture.md b/docs/architecture.md index 0210119..a701575 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -68,7 +68,7 @@ Solid arrows are per-turn data flow; dotted arrows are cloud / conditional. All | **dotty-behaviour** | Docker host | Perception event bus, 11 consumer classes (the running set is config-gated), vision/audio explain endpoints, proactive greeter, calendar context | FastAPI container, port 8090 | | **bridge.py** | Docker host | Admin dashboard service (`/ui`, port 8081). Voice and perception roles were retired in #36; dashboard port to dotty-behaviour is pending. | FastAPI container, port 8081 | | **llama-swap** | Same host or LAN GPU host | Routes OpenAI-compatible requests to per-model llama-server children; co-loads `qwen3.5:4b` (pi outer loop) and `qwen3.6:27b-think` (`think_hard` target) | Docker container (`ghcr.io/mostlygeek/llama-swap:cuda`) | -| **OpenRouter** | Cloud | Routes cloud LLM calls (smart_mode `claude-sonnet-4-6`, VLM `gemini-2.0-flash`, audio caption `gemini-2.5-flash`) | External | +| **OpenRouter** | Cloud | Routes cloud LLM calls (smart_mode `claude-sonnet-4-6`, VLM `gemini-3.1-flash-lite`, audio caption `gemini-2.5-flash`) | External | ## Data flow (single utterance, PiVoiceLLM โ€” normal turn) @@ -122,7 +122,7 @@ sequenceDiagram BH-->>PI: latest cached vision description ``` -The five voice tools in `dotty-pi-ext`: `memory_lookup`, `remember`, `think_hard`, `take_photo`, `play_song`. See [brain.md](./brain.md) for the full tool catalogue. +The seven voice tools in `dotty-pi-ext`: `memory_lookup`, `recall_person`, `remember`, `remember_person`, `think_hard`, `take_photo`, `play_song`. See [brain.md](./brain.md) for the full tool catalogue. ## Why this shape @@ -148,7 +148,7 @@ It does **not** know about llama-swap model names, brain.db, or OpenRouter keys. **dotty-pi (pi agent)** knows: - The llama-swap endpoint and model aliases (via `models.json` inside the container) -- The `dotty-pi-ext` extension with the five voice tools +- The `dotty-pi-ext` extension with the seven voice tools - The persona files and `brain.db` (mounted from host appdata) It does **not** know about the xiaozhi WebSocket protocol or audio. @@ -236,7 +236,7 @@ The canonical working copies live in this repo. | File / Directory | Deployed to | Purpose | |---|---|---| | `dotty-pi/` | Docker host `/mnt/user/appdata/dotty-pi-src/` | pi agent container (Dockerfile + docker-compose.yml) | -| `dotty-pi-ext/` | Docker host (bind-mounted into dotty-pi) | dotty-pi-ext extension โ€” five voice tools | +| `dotty-pi-ext/` | Docker host (bind-mounted into dotty-pi) | dotty-pi-ext extension โ€” seven voice tools | | `dotty-behaviour/` | Docker host `/mnt/user/appdata/dotty-behaviour-src/` | Perception + ambient behaviour container | | `bridge.py` | Docker host (bridge.py container) | Admin dashboard FastAPI service | | `bridge/requirements.txt` | bridge.py container | Pinned Python deps | diff --git a/docs/brain.md b/docs/brain.md index 353a2f1..3e3cae7 100644 --- a/docs/brain.md +++ b/docs/brain.md @@ -9,7 +9,7 @@ description: The pi agent runtime (dotty-pi container), the model matrix, and th - The "brain" is the **`dotty-pi` Docker container** running the pi coding agent with the `dotty-pi-ext` extension. - **`PiVoiceLLM`** (the default xiaozhi LLM provider) translates each voice turn into a pi RPC request via `docker exec -i dotty-pi pi --mode rpc`. TTS-bound text streams back to xiaozhi-server; tool dispatch happens entirely inside the container. -- The `dotty-pi-ext` extension exposes **five voice tools** to the agent loop: `memory_lookup`, `remember`, `think_hard`, `take_photo`, `play_song`. +- The `dotty-pi-ext` extension exposes **seven voice tools** to the agent loop: `memory_lookup`, `recall_person`, `remember`, `remember_person`, `think_hard`, `take_photo`, `play_song`. - **Which LLM runs which turn:** the pi outer loop targets `qwen3.5:4b` (local llama-swap, ~500 ms warm); `think_hard` escalates directly to `qwen3.6:27b-think` (co-resident on llama-swap). **Smart-mode does NOT swap the backend model on the live `PiVoiceLLM` path** โ€” it flips ambient/behaviour only; the inner-loop model-swap is v2 scope and not wired. (Instant in-process model-swap existed only on the now-removed `Tier1Slim` provider.) - One documented alternate voice provider exists: **`OpenAICompat`** (points straight at any OpenAI-compatible endpoint; stateless, no voice tools). See [llm-backends.md](./llm-backends.md). @@ -22,10 +22,10 @@ description: The pi agent runtime (dotty-pi container), the model matrix, and th | PiVoiceLLM outer agent loop | `qwen3.5:4b` | local llama-swap | Every voice turn. ~500 ms warm. | | pi tool: `think_hard` | `qwen3.6:27b-think` | local llama-swap | Multi-step reasoning; direct POST from dotty-pi-ext, no agent overhead. | | pi tool: `memory_lookup` | (no LLM call โ€” FTS5) | brain.db inside dotty-pi | `"do you rememberโ€ฆ"` queries. | -| pi tool: `take_photo` | `google/gemini-2.0-flash-001` (`VLM_MODEL`) | dotty-behaviour โ†’ OpenRouter | Camera describe. | +| pi tool: `take_photo` | `google/gemini-3.1-flash-lite` (`VLM_MODEL`) | dotty-behaviour โ†’ OpenRouter | Camera describe. | | pi tool: `play_song` | (no LLM call) | Firmware via `/xiaozhi/admin/play-asset` | Song request. | | Smart-mode inner loop (`SMART_MODEL`) | `anthropic/claude-sonnet-4-6` | OpenRouter | **Not wired on the live `PiVoiceLLM` path โ€” v2 scope.** Smart-mode flips ambient/behaviour only; it does NOT swap the inner-loop model today. (`SMART_MODEL` is still consumed for dashboard-adjacent calls.) | -| Vision narrative (security/scene synthesis) | `VISION_MODEL` (`google/gemini-2.0-flash-001`) | OpenRouter | dotty-behaviour internal โ€” camera frame description. | +| Vision narrative (security/scene synthesis) | `VISION_MODEL` (`google/gemini-3.1-flash-lite`) | OpenRouter | dotty-behaviour internal โ€” camera frame description. | | Audio captioning (security mode) | `AUDIO_CAPTION_MODEL` (`google/gemini-2.5-flash`) | OpenRouter | dotty-behaviour internal โ€” ambient sound description. | ## The pi agent runtime @@ -48,21 +48,23 @@ Appdata layout on the Docker host: โ”œโ”€โ”€ agent/ โ”‚ โ””โ”€โ”€ models.json # provider config (llama-swap endpoints + aliases) โ”œโ”€โ”€ sessions/ # pi session state -โ”œโ”€โ”€ persona/ # Dotty persona files +โ”œโ”€โ”€ persona/ # operator-owned files; not loaded by live voice RPC โ”œโ”€โ”€ memory/ โ”‚ โ””โ”€โ”€ brain.db # FTS5 full-text store โ””โ”€โ”€ extensions/ โ””โ”€โ”€ dotty-pi-ext/ # voice-tool extension ``` -### dotty-pi-ext โ€” the five voice tools +### dotty-pi-ext โ€” the seven voice tools `dotty-pi-ext` is the pi extension that exposes Dotty's voice tools to the agent loop. Installed inside the container at `/root/.pi/extensions/dotty-pi-ext/`. | Tool | What it does | |---|---| | `memory_lookup(query)` | FTS5 search against `brain.db`; returns top-3 snippets, โ‰ค200 chars each. | +| `recall_person(name)` | Retrieves approved memory facts for a named household member. | | `remember(fact)` | Stores a durable fact (โ‰ค300 codepoints) into `brain.db` with `category=core`, `importance=0.7`. | +| `remember_person(name, fact)` | Stores a durable fact in a named household member's memory namespace. | | `think_hard(question)` | Direct POST to llama-swap `qwen3.6:27b-think` (`enable_thinking=false`, 200-token cap, terse answer). | | `take_photo()` | GET to `dotty-behaviour /api/voice/take_photo` โ€” returns latest cached vision description if โ‰ค30 s old. | | `play_song(name)` | Resolves free-form name against `/xiaozhi/admin/songs` catalogue (60 s cache), then POSTs `/xiaozhi/admin/play-asset`. | @@ -94,15 +96,17 @@ Previously used by the ZeroClawLLM provider via OpenRouter. Not used in the curr Qwen3 is multilingual by training and occasionally **leaks Chinese mid-response** when context is long or system-prompt adherence is weakened by MoE expert routing. Observed symptom: the model starts a response in English and drops a Chinese character or phrase partway through; `en-*` EdgeTTS voices return silent audio on non-English input, making it sound like a dead mic. -**Mitigation in the current stack:** - -1. The pi agent persona (`persona/dotty_voice.md`) has English hard rules. -2. xiaozhi-server's top-level `prompt:` in `data/.config.yaml` is also English-only. -3. `custom-providers/textUtils.py` appends a per-turn English-only suffix (used by PiVoiceLLM). +**Mitigation in the current stack:** `custom-providers/textUtils.py` appends +an English-only suffix to every live PiVoiceLLM turn. Pi is invoked with +`--no-context-files`, and PiVoiceLLM forwards the last user message rather than +xiaozhi-server's configured system dialogue, so neither the operator persona +files nor `.config.yaml`'s top-level `prompt:` is an additional live-path +layer. English-only remains model-enforced; unlike the leading emoji, it does +not have a deterministic output validator. ### qwen3.5:4b (pi outer agent loop) -Local on llama-swap (dual RTX 3060). Fast: ~500 ms warm round-trip including TTS dispatch. Trained for tool calling, which is what lets the five-tool catalogue work reliably at 4 B parameters. See the dotty-pi-ext tool table above. +Local on llama-swap (dual RTX 3060). Fast: ~500 ms warm round-trip including TTS dispatch. Trained for tool calling, which is what lets the seven-tool catalogue work at 4 B parameters. See the dotty-pi-ext tool table above. ### qwen3.6:27b-think (think_hard target) @@ -111,7 +115,7 @@ Local on the same llama-swap, separate alias. ~18 tok/s generation, ~30โ€“50 s c ### Cloud models (smart_mode + visual + audio) - **Smart-mode inner loop:** `anthropic/claude-sonnet-4-6` (`SMART_MODEL` env var). **Not wired on the live `PiVoiceLLM` path** โ€” the inner-loop model-swap is v2 scope; smart-mode currently flips ambient/behaviour only. The env var is still read for dashboard-adjacent calls. -- **VLM (`take_photo`, security camera frames):** `google/gemini-2.0-flash-001` (`VLM_MODEL`). Served by dotty-behaviour. +- **VLM (`take_photo`, security camera frames):** `google/gemini-3.1-flash-lite` (`VLM_MODEL`). Served by dotty-behaviour. - **Audio captioning (security mode):** `google/gemini-2.5-flash` (`AUDIO_CAPTION_MODEL`). Served by dotty-behaviour. ## OpenRouter diff --git a/docs/cookbook/change-persona.md b/docs/cookbook/change-persona.md index 2739691..ceb8800 100644 --- a/docs/cookbook/change-persona.md +++ b/docs/cookbook/change-persona.md @@ -5,14 +5,14 @@ description: Swap Dotty's personality by editing the persona prompt or pointing # Change Persona -Dotty's personality comes from a persona file loaded as the LLM system prompt. **Where** that file lives depends on which LLM provider is active. +Persona-file support depends on which LLM provider is active. Three personas ship in `personas/`: | File | Style | Used by | |---|---|---| | `default.md` | Cheerful, curious desktop robot. The general-purpose persona for generic providers. | `OpenAICompat` | -| `dotty_voice.md` | Voice-tuned variant of `default.md` โ€” same character but pruned for short replies, with the tool catalogue and `[REMEMBER: ...]` markers baked in. | `PiVoiceLLM` | +| `dotty_voice.md` | Voice-tuned reference persona retained for providers or future workflows that load context files. | Not loaded by current PiVoiceLLM | | `smart.md` | More capable, allowed longer answers โ€” for when `smart_mode` is on and the cloud model is doing the heavy lifting. | optional override | ## Which file controls the persona? @@ -21,12 +21,12 @@ Check `selected_module.LLM` in `.config.yaml`, then read the matching block: | Provider | Persona source | |---|---| -| `PiVoiceLLM` (current default) | The persona file configured in the pi agent's extension (`dotty-pi-ext`). Defaults to `personas/dotty_voice.md`. | +| `PiVoiceLLM` (current default) | No persona file. It forwards the last user message plus versioned per-turn policy from `pi_voice.py`/`textUtils.py`; Pi runs with `--no-context-files`. | | `OpenAICompat` (and similar generic providers) | `LLM.OpenAICompat.persona_file` in `.config.yaml`. | ## Switch to a different shipped persona -1. Edit `.config.yaml` (or the pi agent persona config for `PiVoiceLLM`): +1. With `OpenAICompat` selected, edit `.config.yaml`: ```yaml LLM: @@ -38,15 +38,15 @@ Check `selected_module.LLM` in `.config.yaml`, then read the matching block: ## Create your own persona -1. Copy an existing file: `cp personas/dotty_voice.md personas/pirate.md`. +1. Copy an existing file: `cp personas/default.md personas/pirate.md`. 2. Edit the new file. **Keep the emoji instruction line** โ€” the firmware needs it to animate the face. See [emoji-mapping.md](../emoji-mapping.md) for the allowlist (๐Ÿ˜Š๐Ÿ˜†๐Ÿ˜ข๐Ÿ˜ฎ๐Ÿค”๐Ÿ˜ ๐Ÿ˜๐Ÿ˜๐Ÿ˜ด). -3. Point the active provider's `persona_file` at the new file in `.config.yaml`, then restart. +3. Point `OpenAICompat.persona_file` at the new file in `.config.yaml`, then restart. ## Quick inline edit (no file swap) -Edit the top-level `prompt:` block in `.config.yaml` directly. This is the xiaozhi-server system prompt; it gets injected alongside the persona file for most providers. On the `PiVoiceLLM` path the pi agent's own persona file (in `dotty-pi-ext`) is the primary source โ€” edit that for substantive personality changes. +For OpenAICompat, edit the top-level `prompt:` block in `.config.yaml`; xiaozhi includes it with that provider's dialogue. PiVoiceLLM does not forward this dialogue. There is not yet an operator-facing hot-swap workflow for PiVoice personas; changing its behavior requires a reviewed change to the versioned per-turn policy and redeploying xiaozhi-server. ## Notes -- Always keep the emoji-leader rule in any persona โ€” removing it breaks face animations. The persona prompt and the xiaozhi-server system prompt are the two enforcement layers. +- For persona-loading providers, retain the emoji-leader rule. PiVoiceLLM additionally guarantees a neutral leading fallback in code. - See [protocols.md](../protocols.md) for the emoji โ†’ face frame mapping. diff --git a/docs/emoji-mapping.md b/docs/emoji-mapping.md index 58e0c8c..9985b2d 100644 --- a/docs/emoji-mapping.md +++ b/docs/emoji-mapping.md @@ -27,20 +27,17 @@ the corresponding face animation. `custom-providers/textUtils.py`. "Upstream" means it exists in the base xiaozhi-server code. -## Enforcement (no code fallback on the live path) +## Enforcement on the live PiVoiceLLM path -On the live `PiVoiceLLM` path there is **no programmatic emoji fallback**. -The old `bridge.py::_ensure_emoji_prefix()` belonged to the retired ZeroClaw -voice path and is gone. The emoji prefix is enforced entirely by prompt -layers: (1) the pi agent persona prompt (`personas/dotty_voice.md`, loaded by -the `dotty-pi` container), and (2) the top-level `prompt:` block in -`data/.config.yaml` injected by xiaozhi-server. The shared -`custom-providers/textUtils.py` (`build_turn_suffix`, `EMOJI_MAP`, -`get_emotion`) carries the per-turn suffix and the emoji โ†’ emotion lookup. +`build_turn_suffix()` requests one of the nine allowed emojis on every turn. +`PiVoiceLLM._enforce_leading_emoji()` then enforces the wire contract: it +preserves an allowed prefix or prepends neutral `๐Ÿ˜` before TTS. Persona files +and xiaozhi's top-level `.config.yaml` prompt are not forwarded by PiVoiceLLM; +Pi runs with `--no-context-files`. -If the LLM omits the emoji prefix anyway, nothing prepends one โ€” the firmware -receives no emotion frame and keeps its current expression. If the emoji is -not in `EMOJI_MAP`, the same applies. +If the model omits the prefix, the neutral fallback is used. A newly allowed +emoji must be added consistently to `ALLOWED_EMOJIS`, `EMOJI_MAP`, and the +firmware mapping or it will not select the intended face. ## How to Add a New Emoji @@ -51,15 +48,13 @@ See [docs/cookbook/add-emoji.md](cookbook/add-emoji.md). | Component | File | What it does | |-----------|------|-------------| | Per-turn emoji + rules suffix | `custom-providers/textUtils.py` | `build_turn_suffix()` (appended on the live `PiVoiceLLM` path) | +| Leading-emoji enforcement | `custom-providers/pi_voice/pi_voice.py` | `_enforce_leading_emoji()` | | Emoji โ†’ emotion | `custom-providers/textUtils.py` | `EMOJI_MAP` dict, `get_emotion()` | -| Persona emoji rule | `personas/dotty_voice.md` | loaded by the `dotty-pi` agent | -| xiaozhi system prompt | `data/.config.yaml` | top-level `prompt:` block | | Emotion โ†’ face | StackChan firmware | Avatar renderer, expression assets | ## Upstream Emojis Not Used by Dotty The upstream `EMOJI_MAP` includes additional emojis that Dotty doesn't use in its 9-emoji set: ๐Ÿ˜‚ ๐Ÿ˜ญ ๐Ÿ˜ฒ ๐Ÿ˜ฑ ๐Ÿ˜Œ ๐Ÿ˜œ ๐Ÿ™„ ๐Ÿ˜ถ ๐Ÿ™‚ ๐Ÿ˜ณ ๐Ÿ˜‰ ๐Ÿ˜Ž ๐Ÿคค ๐Ÿ˜˜ ๐Ÿ˜. -These would work if the LLM produced them (the firmware would show the -face), but the persona prompt and the `.config.yaml` `prompt:` block -constrain responses to the 9 emojis in the active mapping above. +PiVoiceLLM's per-turn suffix constrains responses to the nine emojis above, +and its `ALLOWED_EMOJIS` check replaces any other leading emoji with `๐Ÿ˜`. diff --git a/docs/faq.md b/docs/faq.md index f4434ca..bc1341f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -55,8 +55,8 @@ With Piper TTS and the default local model, nothing leaves your LAN. The trade-o **Kid Mode is ON by default** (`DOTTY_KID_MODE=true`). It applies age-appropriate prompt steering plus a thin blocked-words filter on spoken output (see the honest caveat below). You can disable it with `DOTTY_KID_MODE=false` for general-purpose use. What Kid Mode enforces: -- Per-turn sandwich enforcement forces the LLM to respond in English with an emoji prefix, which limits the scope of unexpected output. -- The persona prompt (`personas/dotty_voice.md`) defines the robot's personality and boundaries with kid-safe defaults. +- The per-turn sandwich instructs the LLM to respond in English and Kid Mode style; code guarantees the emoji prefix, while English and topic adherence remain model-enforced. +- PiVoiceLLM's versioned per-turn suffix defines the live child-safety and response-style policy. - Content and tone are constrained to be age-appropriate. - A blocked-words filter runs on TTS-bound LLM output ([#157](https://github.com/BrettKinny/dotty-stackchan/issues/157)): if a reply matches the blocklist (profanity, explicit content, hard drugs), the turn is replaced with a cheerful redirect before it's spoken. The same blocklist guards the dashboard say/story ingresses. @@ -71,9 +71,7 @@ This is a self-hosted system โ€” you control the prompt, the model, and every lo ### Can I change the robot's personality? -Yes. The persona is a Markdown file โ€” `personas/dotty_voice.md`, loaded by the active LLM provider. Edit it and restart the relevant container. - -There's also a secondary `prompt:` key in `data/.config.yaml` that gets injected as a system message โ€” a useful place for voice-pipeline-level hints. Full instructions: [cookbook/change-persona.md](./cookbook/change-persona.md). +It depends on the provider. `OpenAICompat` loads its configured Markdown persona and xiaozhi system dialogue. The default PiVoiceLLM currently does neither: it sends the last user message plus the versioned policy in `pi_voice.py`/`textUtils.py`. See [change persona](./cookbook/change-persona.md) for the supported options and current limitation. --- diff --git a/docs/kid-mode.md b/docs/kid-mode.md index 88c23f8..e01b5bb 100644 --- a/docs/kid-mode.md +++ b/docs/kid-mode.md @@ -29,9 +29,9 @@ asks). Only the child-specific rules (4-9) are removed. ### Hot-reload (no daemon restart) -Both the bridge dashboard's `POST /admin/kid-mode` endpoint and the dashboard toggle persist the new value and call `_apply_kid_mode(enabled)`, which re-binds the dashboard's kid-mode globals (`KID_MODE`, `VOICE_TURN_SUFFIX` via `build_turn_suffix(enabled)`). **No dashboard restart is required** to flip the persisted value at runtime. (This is the dashboard's own state; the live voice path reads kid-mode independently โ€” see below.) +Both the bridge dashboard's `POST /admin/kid-mode` endpoint and the dashboard toggle persist the new value to the shared `DOTTY_KID_MODE_STATE` file and call `_apply_kid_mode(enabled)`, which re-binds the dashboard's kid-mode globals (`KID_MODE`, `VOICE_TURN_SUFFIX` via `build_turn_suffix(enabled)`). **No dashboard restart is required** to flip the persisted value at runtime. -The xiaozhi-server side of kid-mode lives in the active LLM provider's persona / suffix. On the live `PiVoiceLLM` path, `pi_voice.py` reads kid-mode as a process-start snapshot and bakes it into the suffix produced by `build_turn_suffix(kid_mode)`; the persona is loaded per-session by the `dotty-pi` agent. A persona/topic change lands on the next turn, while flipping the kid-mode snapshot itself requires a container restart to re-read the value into the live provider instance. +The xiaozhi-server container mounts the same state file read-only. On the live `PiVoiceLLM` path, `pi_voice.py` re-reads it at the start of every voice turn and passes the result to both `build_turn_suffix(kid_mode)` and the output filter. A dashboard toggle therefore changes the live voice guardrails on the next turn without restarting either container. If the state file is absent, unreadable, or malformed, the provider falls back to `DOTTY_KID_MODE` (which defaults to `true`). Pi is invoked with `--no-context-files`, so files in the dotty-pi persona directory are not loaded into live voice turns. ## Guardrail details @@ -40,44 +40,25 @@ enforcement code lives, and what gaps remain. --- -## Architecture: Three-Layer Sandwich Enforcement +## Architecture: Live Runtime Enforcement -Every voice turn passes through three independent layers before reaching the -speaker. Each layer reinforces the same rules so that a failure in one layer -is caught by the next. +Every live `PiVoiceLLM` turn uses one versioned prompt-policy layer followed +by deterministic output backstops where those backstops are implemented. > **Layering on the live `PiVoiceLLM` path:** -> - **Layer 1** is `personas/dotty_voice.md` (loaded by the `dotty-pi` agent). -> - **Layer 2** is the `prompt:` block in `.config.yaml` injected by xiaozhi-server. -> - **Layer 3** is the per-turn **sandwich suffix** โ€” `build_turn_suffix(kid_mode)` from `custom-providers/textUtils.py`, applied by `custom-providers/pi_voice/pi_voice.py` (`_wrap_with_sandwich`). This **ships on the live path** and includes the kid-mode topic constraints (rules below) when kid-mode is on. +> - **Prompt policy** is the per-turn **sandwich suffix** โ€” `build_turn_suffix(kid_mode)` from `custom-providers/textUtils.py`, applied by `custom-providers/pi_voice/pi_voice.py` (`_wrap_with_sandwich`). It includes the kid-mode topic constraints (rules below) when kid-mode is on. +> - **Emoji backstop** is `_enforce_leading_emoji()` in `pi_voice.py`, which guarantees an allowed leading face glyph independently of model compliance. > - **Output backstop:** `filter_tts_stream()` in `custom-providers/textUtils.py` buffers the complete Kid Mode reply, checks the shared blocked-words tiers, and replaces a matching turn before TTS. Both `PiVoiceLLM` and `OpenAICompat` use it. This is a thin, bypassable word-level backstop, not a content-safety guarantee; live red-team verification remains tracked in #157. > +> PiVoiceLLM forwards only the last user message plus this per-turn policy. Its +> Pi command uses `--no-context-files`; neither `personas/dotty_voice.md` nor +> xiaozhi-server's top-level `.config.yaml` `prompt:` is injected into this RPC +> request. Those files may apply to other providers but are not live +> PiVoiceLLM enforcement layers. +> > The `Tier1Slim` provider was removed entirely and is no longer a live or rollback option. -### Layer 1 -- Agent Persona Prompt (dotty-pi container) - -The `dotty-pi` agent's persona prompt (`personas/dotty_voice.md`) sets the baseline: stay cheerful, -age-appropriate, begin every reply with an emoji. This is the "inner" system -prompt that the LLM sees at the top of its context. - -### Layer 2 -- xiaozhi-server System Prompt (server) - -The `prompt:` block in `.config.yaml` is injected by xiaozhi-server as a -system message. It reinforces the emoji rule and the short-sentence, -TTS-friendly style. Relevant excerpt: - -```yaml -prompt: | - You are , a small desktop robot assistant for a curious family - with young children. - ... - Critical output rules: - - ALWAYS begin your reply with exactly one emoji that conveys your emotion. - - Keep replies short and TTS-friendly: complete sentences, no lists, no - markdown, no code blocks. -``` - -### Layer 3 -- Per-Turn Sandwich Suffix (`build_turn_suffix` in `textUtils.py`) +### Per-Turn Sandwich Suffix (`build_turn_suffix` in `textUtils.py`) On the live `PiVoiceLLM` path, every turn has a suffix appended before being sent to the LLM: @@ -96,10 +77,10 @@ the hardest to override. When `kid_mode` is true the suffix carries the full child-safe topic constraints (rules 4-9 below); when false, only the English-only / emoji-leader / length rules remain. -**Why a suffix, not just a system prompt?** System prompts are seen once and -can be diluted by long conversations. The suffix is re-injected on every -single turn, and its position at the end of the context window gives it -disproportionate influence on the model's output. +**Why a suffix?** PiVoiceLLM does not forward xiaozhi's system dialogue and +disables Pi context files. The suffix is therefore the versioned policy that +is re-injected on every turn, and its position at the end of the prompt gives +it disproportionate influence on the model's output. ### Post-generation blocked-words backstop @@ -219,16 +200,17 @@ additional layers are needed). The emoji that begins each reply is not decorative -- the StackChan firmware parses it into a facial expression on the robot's screen. If the emoji is -missing, the face stays blank. Three layers enforce it: +missing, the face stays blank. The live path has two enforcement points: -1. **Agent persona prompt** (`personas/dotty_voice.md`, loaded by `dotty-pi`) -- tells the model to begin with an emoji. -2. **xiaozhi-server system prompt** (`.config.yaml` `prompt:` block) -- - repeats the rule with the exact emoji set. -3. **Per-turn suffix rule 2** (`build_turn_suffix` in `custom-providers/textUtils.py`) -- restates the exact emoji set at the end of every turn. +1. **Per-turn suffix rule 2** (`build_turn_suffix` in `custom-providers/textUtils.py`) instructs the model with the exact emoji set at the end of every turn. +2. **Programmatic output enforcement** (`_enforce_leading_emoji()` in `custom-providers/pi_voice/pi_voice.py`) guarantees an allowed leading glyph. -There is **no programmatic emoji fallback** on the live path. The old -`_ensure_emoji_prefix` was ZeroClaw-only and is gone; the three prompt layers -above are load-bearing. +`PiVoiceLLM` also enforces the contract programmatically before its output +reaches the Kid Mode content filter or TTS. `_enforce_leading_emoji()` buffers +leading whitespace, preserves an allowed face emoji, and replaces a missing +or disallowed leading emoji with the neutral `๐Ÿ˜` fallback. The per-turn prompt +remains the primary instruction; this output guard is the deterministic +backstop. Allowed emojis and their face mappings: @@ -244,10 +226,8 @@ Allowed emojis and their face mappings: | ๐Ÿ˜ | love | | ๐Ÿ˜ด | sleepy | -Error responses on the live `PiVoiceLLM` path are plain text (e.g. -`(brain offline โ€” try again in a moment)` in `pi_voice.py`); they are not -forced to carry an emoji, since the ZeroClaw fallback that prepended `๐Ÿ˜` is -gone. +Error and empty responses on the live `PiVoiceLLM` path also carry the neutral +face prefix, for example `๐Ÿ˜ (brain offline โ€” try again in a moment)`. --- @@ -255,7 +235,7 @@ gone. When things go wrong, the system defaults to a safe canned reply rather than exposing raw error text or going silent. On the live `PiVoiceLLM` path the -`dotty-pi`-unavailable case yields `(brain offline โ€” try again in a moment)` +`dotty-pi`-unavailable case yields `๐Ÿ˜ (brain offline โ€” try again in a moment)` (hardcoded in `custom-providers/pi_voice/pi_voice.py`), independent of LLM cooperation. The detailed per-failure-mode emoji-prefixed canned replies listed in earlier docs belonged to the retired ZeroClaw bridge and no longer @@ -283,19 +263,17 @@ inappropriate content through). ## Where the Code Lives -The live `PiVoiceLLM` path layers the persona prompt and the per-turn sandwich -suffix. There is no live bridge involvement. +The live `PiVoiceLLM` path uses the per-turn sandwich and output backstops. +There is no live bridge prompt or persona-file involvement. | Component | File | Symbol | |---|---|---| | Per-turn sandwich suffix (the live sandwich) | `custom-providers/textUtils.py` | `build_turn_suffix(kid_mode)` | | Sandwich injection on the voice path | `custom-providers/pi_voice/pi_voice.py` | `_wrap_with_sandwich()` (calls `build_turn_suffix`) | | Emoji โ†’ emotion lookup | `custom-providers/textUtils.py` | `EMOJI_MAP`, `get_emotion()` | -| dotty-pi-unavailable canned reply | `custom-providers/pi_voice/pi_voice.py` | `(brain offline โ€” try again in a moment)` | -| xiaozhi system prompt | `data/.config.yaml` | Top-level `prompt:` block | -| Agent persona prompt | `personas/dotty_voice.md` | loaded by the `dotty-pi` agent | +| dotty-pi-unavailable canned reply | `custom-providers/pi_voice/pi_voice.py` | `๐Ÿ˜ (brain offline โ€” try again in a moment)` | | Blocked-words content filter | `custom-providers/textUtils.py` | `content_filter_match()`, `filter_tts_stream()`; shared by both live LLM providers and enabled only in Kid Mode. | -| Emoji-prefix fallback | โ€” | **Absent.** Was `_ensure_emoji_prefix()` in the retired ZeroClaw bridge; prompt layers are now load-bearing. | +| Emoji-prefix fallback | `custom-providers/pi_voice/pi_voice.py` | `_enforce_leading_emoji()` | --- @@ -337,7 +315,7 @@ is to route the `stackchan` channel to a model with stronger built-in safety ### Modifying the Topic Blocklist -Edit the suffix text in `build_turn_suffix()` in `custom-providers/textUtils.py` (rule 5), and/or edit rule 5 in `personas/dotty_voice.md`. After editing, restart the xiaozhi-server container. +Edit rule 5 in `build_turn_suffix()` in `custom-providers/textUtils.py`, then redeploy or restart the xiaozhi-server container. ### Changing the Self-Harm Response @@ -348,7 +326,7 @@ wording was chosen to acknowledge distress without attempting counseling. 1. Update rule 2 in `build_turn_suffix()` (`custom-providers/textUtils.py`) to add or remove emojis. 2. Update `EMOJI_MAP` in `custom-providers/textUtils.py` so the new emoji maps to an emotion. -3. Update the `prompt:` block in `data/.config.yaml` and the persona prompt to match. +3. Update `ALLOWED_EMOJIS` in `custom-providers/textUtils.py`, which controls the programmatic prefix check. 4. Confirm the StackChan firmware supports the face mapping for any new emoji. ### Changing the Age Range @@ -361,9 +339,9 @@ adjusting downward would further simplify language. ## Design Principles -- **Defense in depth.** No single layer is trusted alone. The persona prompt, - the xiaozhi system prompt, and the per-turn sandwich suffix each - independently restate the core rules. +- **Defense in depth where deterministic checks exist.** The per-turn suffix + steers all safety and style rules; deterministic code additionally enforces + the leading emoji and blocks a small set of output terms in Kid Mode. - **Fail safe, not fail open.** Error paths produce a safe canned reply rather than raw error text or stack traces reaching the speaker. - **Suffix position is deliberate.** Placing the hard constraints at the end diff --git a/docs/llm-backends.md b/docs/llm-backends.md index fbbe0b7..77888be 100644 --- a/docs/llm-backends.md +++ b/docs/llm-backends.md @@ -19,7 +19,7 @@ and the matching block under `LLM:` in `.config.yaml`. | **Cost** | Pay-per-token | Free (electricity + hardware) | Free (electricity + hardware) | | **Privacy** | Tokens sent to cloud provider | Fully local, nothing leaves LAN | Fully local | | **Setup complexity** | Low โ€” API key + model name | Medium โ€” GPU, Docker, GGUF download | Medium โ€” dotty-pi container + llama-swap | -| **Memory / tools** | None | None | Yes โ€” memory_lookup, remember, think_hard, take_photo, play_song | +| **Memory / tools** | None | None | Yes โ€” memory_lookup, recall_person, remember, remember_person, think_hard, take_photo, play_song | | **Hot-swappable** | Restart container | Restart container | Restart container | | **Best for** | Quick start, best-in-class models | Privacy + concurrent multi-model serving | **Default โ€” snappy voice with full tool support** | @@ -132,17 +132,20 @@ LLM: The default in the shipped `.config.yaml`. The `PiVoiceLLM` provider routes each voice turn to the **dotty-pi container** โ€” the pi coding agent running on the same Docker host as xiaozhi-server. -`PiClient` drives the agent by running `docker exec -i dotty-pi pi --mode rpc โ€ฆ` and exchanging JSONL messages over its stdin/stdout. The agent's outer loop uses `qwen3.5:4b` on local llama-swap for fast chitchat and loads the **dotty-pi-ext extension**, which exposes five voice-focused tools: +`PiClient` drives the agent by running `docker exec -i dotty-pi pi --mode rpc โ€ฆ --no-context-files` and exchanging JSONL messages over its stdin/stdout. The agent's outer loop uses `qwen3.5:4b` on local llama-swap for fast chitchat and loads the **dotty-pi-ext extension**, which exposes seven voice-focused tools: | Tool | Purpose | |---|---| | `memory_lookup` | Recall a fact from past conversations (FTS on brain.db) | +| `recall_person` | Recall approved facts about a named household member | | `remember` | Stash a new fact into brain.db | +| `remember_person` | Store a fact about a named household member | | `think_hard` | Escalate a hard question to `qwen3.6:27b-think` | | `take_photo` | Describe what Dotty's camera sees via a VLM | | `play_song` | Play a song through the speaker | Only TTS-bound text streams back to xiaozhi-server โ€” tool results stay internal to the agent loop. +PiVoiceLLM does not use `persona_file` or forward xiaozhi's configured system dialogue; its live policy is appended to every user turn by the provider. ### Prerequisites diff --git a/docs/modes.md b/docs/modes.md index e570347..4358dff 100644 --- a/docs/modes.md +++ b/docs/modes.md @@ -43,7 +43,7 @@ The firmware boots into `idle` with both toggles **off**. The bridge resyncs tog | `story_time` | warm `(100,40,0)` | NORMAL | Long-running interactive story. | Phase 7 PENDING โ€” backing path unimplemented | | `security` | white `(80,80,80)` **flashing 1 Hz** across all 6 left pixels (`kSecurityFlashHalfMs = 500`) | SURVEILLANCE | Wide deliberate scan, serious face, periodic photo + audio capture. No proactive greet. | Phase 8 PENDING โ€” `SecurityCycle` consumer is scaffolding, not a live path | | `sleep` | very dim blue `(0,0,16)` | SLEEPY | Head face-down + centred, servo torque off (with `kSleepTorqueReleaseTimeoutMs = 3000` fallback), sleeping emoji on screen, ambient awareness paused. Wakes on face / voice / head-pet. | firmware-only quiescence (Phase 5) | -| `dance` | rainbow sweep (left ring) | NORMAL | Transient performance โ€” choreography + audio. Pre-existing dance handler. | `receiveAudioHandle.py::_handle_dance` | +| `dance` | rainbow sweep (left ring) | NORMAL | Transient performance โ€” server-owned named choreography + audio. Firmware holds the cooperative motion lock so face tracking and idle motion cannot overwrite MCP keyframes. | `receiveAudioHandle.py::_handle_dance` | The `idle โ†’ talk` trigger is the firmware `face_detected` event (any face, family or stranger) **or** `onVoiceListening` (the WS opens for a wake-word / inject-text / head-pet hold). dotty-behaviour runs VLM recognition in parallel and feeds the resulting identity into the speaker resolver / persona โ€” recognition does **not** gate the state transition. @@ -193,7 +193,7 @@ Both `kid_mode` and `smart_mode` are voice-untoggleable โ€” they are guardian-co | `story_time` | Phase 7 PENDING โ€” backing path unimplemented | n/a (pending) | n/a (pending) | | `security` | Phase 8 PENDING โ€” `SecurityCycle` consumer is scaffolding, no live path | n/a (pending) | n/a (pending) | | `sleep` | mic stays on for "wake up"; no LLM round-trip | n/a | n/a | -| `dance` | bridge handler dispatches choreography + audio file | n/a | dance MCP | +| `dance` | xiaozhi handler dispatches choreography + audio file; firmware state supplies exclusive motion ownership, not a second choreography | n/a | head/LED MCP | `smart_mode` is a toggle only and sticky across turns; the backing model-swap it was designed to drive is v2 scope and not wired on the `PiVoiceLLM` path. `story_time` (when implemented) would be the only voice path with its own session memory (Phase 7 pending). diff --git a/docs/protocols.md b/docs/protocols.md index ba2c64d..433b8e0 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -246,7 +246,7 @@ xiaozhi-server Each turn is a single JSONL object written to stdin; the agent streams JSONL response chunks back on stdout. Only TTS-bound text chunks are forwarded to xiaozhi-server โ€” tool call details stay internal to the agent loop. The agent exits cleanly after each turn; `PiClient` re-invokes `docker exec` for the next turn. -The dotty-pi agent loads the **dotty-pi-ext extension** at startup, which registers the five voice tools (`memory_lookup`, `remember`, `think_hard`, `take_photo`, `play_song`). Tool results never appear in the TTS stream. +The dotty-pi agent loads the **dotty-pi-ext extension** at startup, which registers seven voice tools (`memory_lookup`, `recall_person`, `remember`, `remember_person`, `think_hard`, `take_photo`, `play_song`). Tool results never appear in the TTS stream. ## HTTP APIs diff --git a/docs/quickstart.md b/docs/quickstart.md index 790b47a..f2b1895 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -183,14 +183,14 @@ This repo uses placeholders in place of real IPs, usernames, and filesystem path | `` | SSH user for the server (whatever your distro defaults to: `root`, `ubuntu`, `dietpi`, etc.). | | `` | Hostname or Tailscale name of the server (optional, IP works for everything). | | `` | Path on the server where you clone/install this repo (e.g. `/opt/xiaozhi-server/` or `/srv/xiaozhi-server/`). | -| `` | Your name / org, used in the persona prompt in `.config.yaml`. | -| `` | Name the robot introduces itself as, referenced in the persona prompt in `.config.yaml`. Any string โ€” pick whatever you want. The default example uses the hardware name ("StackChan"). | +| `` | Your name / org, used by the `.config.yaml` prompt for providers that forward xiaozhi's system dialogue. | +| `` | Robot name used by that configured prompt. PiVoiceLLM's live per-turn policy is versioned separately. | Port numbers (`8000`, `8003`, `8081`, `8090`) are product-generic and should not be changed unless you also reconfigure the respective services. Files you will definitely need to edit before first run: -- `.config.yaml` โ€” replace `` and customise the `prompt:` block. +- `.config.yaml` โ€” replace ``; customise `prompt:` when using a provider such as OpenAICompat that forwards it. - `docker-compose.yml` โ€” set `TZ` to your timezone. --- @@ -272,7 +272,7 @@ curl http://:8081/health The default TTS is `LocalPiper` (offline, runs inside the container). To change the Piper voice, edit `TTS.LocalPiper.voice` and the corresponding `model_path` / `config_path` in `data/.config.yaml`. To switch to cloud EdgeTTS instead, set `selected_module.TTS: EdgeTTS` and edit `TTS.EdgeTTS.voice` (any Microsoft Edge Neural voice ID works, e.g. `en-US-AvaNeural`). Restart the container after changes. ### Changing persona (the robot's personality) -Edit `personas/dotty_voice.md` (loaded by the pi agent on the `PiVoiceLLM` path) and restart the relevant container. The `prompt:` key in `data/.config.yaml` is also injected as a secondary system message. Full instructions: [cookbook/change-persona.md](cookbook/change-persona.md). +`OpenAICompat` supports `persona_file` and the xiaozhi system prompt. The default PiVoiceLLM currently uses neither; its live voice policy is the versioned per-turn prompt in `custom-providers/pi_voice/pi_voice.py` and `custom-providers/textUtils.py`. Full instructions and limitations: [cookbook/change-persona.md](cookbook/change-persona.md). ### Changing VAD sensitivity `VAD.SileroVAD.min_silence_duration_ms` in `data/.config.yaml`. Default: 700 ms. Lower = cuts off quicker. Higher = waits longer for slow speakers. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c34b172..e87850f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -16,8 +16,8 @@ Symptom-first lookup table covering common and obscure failure modes. Pair with **Cause:** Language mismatch between the TTS voice and the response text. EdgeTTS `en-*` voices return empty audio when given non-English text (Chinese, Japanese, etc.). This is not a throttle or rate limit โ€” it's a silent failure in the EdgeTTS service. **Fix:** -1. Check the xiaozhi-server logs for the LLM response text. If it contains non-English characters, the LLM is ignoring the English enforcement in the persona prompt and the `.config.yaml` `prompt:` block. -2. Confirm English enforcement is active: check `personas/dotty_voice.md` and the top-level `prompt:` in `data/.config.yaml` both contain explicit English-only instructions. +1. Check the xiaozhi-server logs for the LLM response text. On PiVoiceLLM, non-English text means the model ignored the per-turn `build_turn_suffix()` policy. +2. Confirm `custom-providers/pi_voice/pi_voice.py` appends `build_turn_suffix()` to the prompt. PiVoiceLLM uses `--no-context-files` and does not forward xiaozhi's system dialogue, so persona and `.config.yaml` prompt edits do not diagnose this provider. 3. Check `data/.config.yaml` to confirm the TTS voice matches the expected response language (e.g., `en-AU-WilliamNeural` for English). 4. If using Piper TTS instead of EdgeTTS, confirm the selected voice model matches the response language. @@ -27,14 +27,13 @@ Symptom-first lookup table covering common and obscure failure modes. Pair with **Symptom:** The robot speaks, but in the wrong language. Logs show Chinese or Japanese text in the LLM response. -**Cause:** The LLM (Qwen3) is ignoring the system prompt's language constraint. This is a known weakness โ€” Qwen3 tends to leak Chinese on long-context English-only prompts, especially mid-session. +**Cause:** The LLM (Qwen3) is ignoring the per-turn language constraint. Qwen3 can leak Chinese despite the English-only suffix; English has no deterministic output validator. **Fix:** 1. Confirm the per-turn sandwich enforcement is active on the live `PiVoiceLLM` path. Static system prompts alone are not enough โ€” every turn is wrapped with an explicit English+emoji suffix. This is `build_turn_suffix()` in `custom-providers/textUtils.py`, applied by `custom-providers/pi_voice/pi_voice.py` (`_wrap_with_sandwich`). (The old `bridge.py::_build_sandwich_prompt` was part of the retired ZeroClaw bridge and no longer exists โ€” the bridge is not in the voice path.) -2. Confirm the persona prompt reinforces English-only: check `personas/dotty_voice.md` (loaded by the `dotty-pi` agent) and the top-level `prompt:` block in `data/.config.yaml`. -3. Watch the actual voice path while testing โ€” tail the `dotty-pi` container logs (`docker logs -f dotty-pi`) and the xiaozhi-server logs, not the bridge. -4. If the leak happens on the first turn, check the persona file (`personas/dotty_voice.md`) for any non-English text. -5. As a last resort, the ASR may be mis-transcribing English as another language. Check the `ASR.FunASR.language` key in `data/.config.yaml` is set to `en` (not `auto`). +2. Inspect the prompt captured by PiVoiceLLM or its unit tests; do not use persona files or the top-level `.config.yaml` prompt as evidence for this path. +3. Watch the actual voice path in xiaozhi-server logs. `docker logs dotty-pi` is normally empty because Pi runs as a `docker exec` child; PiClient owns that child's stdio. +4. As a last resort, the ASR may be mis-transcribing English as another language. Check the configured ASR language is `en` rather than `auto`. --- @@ -111,9 +110,9 @@ Symptom-first lookup table covering common and obscure failure modes. Pair with | `๐Ÿ˜ด` | Sleepy | **Fix:** -1. Check the xiaozhi-server logs for the raw LLM response. Two enforcement layers apply: (a) the configured persona prompt (`personas/dotty_voice.md`), (b) the `prompt:` key in `data/.config.yaml`. If the response still has no emoji after both layers, something is fundamentally wrong with the response path. +1. Check the xiaozhi-server logs for the raw LLM response. PiVoiceLLM's per-turn suffix requests an allowed emoji and `_enforce_leading_emoji()` prepends neutral `๐Ÿ˜` when it is absent. A prefix-free response therefore indicates the deployed provider is stale or bypassed. 2. If the response has an emoji but the face doesn't change, it may be an unsupported emoji. Only the nine listed above are mapped to animations. -3. On the `PiVoiceLLM` path the `_ensure_emoji_prefix` fallback in `bridge.py` is not active โ€” emoji enforcement relies entirely on the persona prompt and the `.config.yaml` `prompt:` block. +3. Confirm the selected provider is PiVoiceLLM and the deployed `pi_voice.py` contains `_enforce_leading_emoji()`. The retired bridge fallback is unrelated. --- diff --git a/docs/uat-runbook.md b/docs/uat-runbook.md index 81ef6e4..e97e17a 100644 --- a/docs/uat-runbook.md +++ b/docs/uat-runbook.md @@ -71,7 +71,7 @@ issues rather than spawning new ones, and their clips are candidate | `story_time` backing path | Phase 7 pending โ€” state/LED rails only | modes.md Phase 7 | | `security` capture path | SecurityCycle scaffolding; audio leg unshipped (#31) | modes.md Phase 8, #31 | | `smart_mode` model swap | toggle-only; swap is v2 scope | #36 cutover notes | -| Tools-inventory card count | dashboard card lists 5 tools; 7 ship | file/update issue in wrap-up | +| Tools-inventory card count | dashboard card may list 5 legacy entries; Pi registers 7 voice tools | treat the dashboard as stale presentation, verify Pi inventory separately | --- @@ -115,7 +115,7 @@ issues rather than spawning new ones, and their clips are candidate | ID | Brett does (on camera) | Expected โ€” eyes/video | Expected โ€” logs (Claude) | Shorts framing | |---|---|---|---|---| -| UT1 | Tell her a keepable fact: *"my favourite colour is purple"* | Warm acknowledgement | `[REMEMBER:]` marker โ†’ `remember` write to brain.db (`category=core`) | Part 1 of the memory two-parter | +| UT1 | Tell her a keepable fact: *"please remember my favourite colour is purple"* | Warm acknowledgement after the write | `remember` tool call โ†’ brain.db write (`category=core`) | Part 1 of the memory two-parter | | UT2 | New turn (or later in session): *"what did I tell you about my favourite colour?"* | She recalls purple | `memory_lookup` tool call + FTS5 hit | Part 2 โ€” "she actually remembered" payoff | | UT3 | *"Remember that Brett loves flat whites"* (adult) | Confirms she'll remember | `remember_person` โ†’ behaviour `person_review_status` โ†’ `person:` store | "Dotty is learning who I am" | | UT4 | Same as UT3 but for a **kid's** name | Confirms, but fact goes to the **pending** queue, not live | classifier routes to `person_pending:`; visible later in dashboard Memory card (UD5) | QA-critical (child-safety path); clip optional | @@ -155,11 +155,11 @@ face events and survive chat-turn ends; `wake up` / `come back` / |---|---|---|---|---| | UL1 | Dashboard: kid_mode ON, then OFF | Pixel 8 warm pink โ‰ค1 s; dashboard dot matches; offโ†’dark | bridge โ†’ `/xiaozhi/admin/set-toggle`; `_apply_kid_mode()` hot-reload | "Kid mode: one pink light" | | UL2 | With kid_mode ON, ask a borderline question (e.g. about a scary movie) | Kind redirect, age-appropriate | content-filter sandwich; Safety card hit (UD7) | QA-critical; clip only if the redirect is charming | -| UL3 | With kid_mode ON: *"what do you see?"* | Camera tool **denied** โ€” she declines gracefully | camera-tool denial in kid persona/tool policy | "Kid mode means no photos, ever" โ€” good trust content | +| UL3 | With kid_mode ON: *"what do you see?"* **โš  known gap** | Record actual behavior; no live PiVoice camera-denial policy exists yet | verify whether shortcut or `take_photo` ran; file a safety issue if camera access occurs | QA-critical; do not present as a shipped privacy guarantee | | UL4 | Dashboard: smart_mode ON, then OFF **โš  pending** | Pixel 9 orange; dot matches; **no behaviour change** (swap is v2) | `set-toggle smart_mode`; `model_swap_active=False` | QA-only | | UL5 | During UL1+UL4, camera close on pixels 7 and 10 | Both stay dark throughout (reserved, locked off) | โ€” | QA-only | | UL6 | Voice: *"turn your LEDs blue"* | Only **left** ring goes blue; right-ring pips untouched | `set_led_color` tool call | "She won't let *anyone* touch her status lights" | -| UL7 | Ask her to set LED **6** red | Pixel 6 unchanged; she demurs (persona: "my lights show how I'm feeling") | serial warn: `set_led_multi: index 6 not on left ring` | tail of UL6 clip | +| UL7 | Ask her to set LED **6** red | Pixel 6 unchanged; record the verbal response | serial warn if firmware receives `set_led_multi`; PiVoice has no LED voice tool | tail of UL6 clip | | UL8 | The combined-indicator stress test: kid ON + smart ON + face identified (green 6) + trigger dance + speak | Rainbow on the left; right ring holds all four pips (6 green in its 4 s window, 8 pink, 9 orange, 11 red); 7+10 dark; โ‰ค200 ms flicker OK (5 Hz re-assert) | `state_changed` โ†’ `dance` | "Every light on at once" โ€” satisfying finale | **โ˜• Break point.** Leave capture + screen recording running (or stop and @@ -202,7 +202,7 @@ dashboard works well for Shorts). | UD3 | Idle โ†’ **Emojis** row: press several of the 9 | Robot's face changes per press | `/ui/actions/mood` | "A remote control for moods" | | UD4 | Talk โ†’ **Say** box: type a line, send | Robot speaks it verbatim | `/ui/actions/say` (โ‰ค500 chars) | "Making my robot say things" โ€” obvious fun | | UD5 | Memory card: find UT4's pending kid fact โ†’ **approve** it; **redact** another | Pending โ†’ approved queue move; redacted fact gone; `recall_person` now sees the approved one | `/ui/actions/memory/{approve,redact}` | QA-critical (human-review loop); clip optional | -| UD6 | Tools-inventory card | **Known stale:** lists 5 tools, 7 ship โ€” record FAIL | โ€” | QA-only | +| UD6 | Tools-inventory card | If it lists 5 legacy entries, record presentation FAIL; separately verify Pi registers all 7 voice tools | Pi RPC/tool-startup evidence is authoritative | QA-only | | UD7 | Safety card after UL2 | The kid-mode filter hit from UL2 listed (last 20) | `/ui/safety/recent` | QA-only | | UD8 | Activity feed: cycle All / Turns / Events / Errors chips; open the errors modal | Live SSE turns + perception events streaming; errors modal renders | `/ui/events` SSE + `/api/perception/feed`; `/ui/alerts/detail` | B-roll: the feed scrolling during a chat | | UD9 | Perception card + vision modal: open latest photo large, download | Latest VLM photo + description, audio caption, last voice line, scene sentence | `/ui/vision/large`, `/ui/vision/photo?download=1` | "What my robot sees" | diff --git a/docs/voice-pipeline.md b/docs/voice-pipeline.md index e93772a..79e023e 100644 --- a/docs/voice-pipeline.md +++ b/docs/voice-pipeline.md @@ -129,7 +129,7 @@ xiaozhi-server doesn't run an emotion classifier. It **strips the leading emoji* The TTS provider receives text **with the emoji already stripped**. The device receives the emotion and sets the face animation; the speaker plays the clean text. -**Surprising consequence**: the LLM must emit the emoji as its very first character for emotion dispatch to fire. On the `PiVoiceLLM` path, enforcement relies on the persona prompt and the `.config.yaml` `prompt:` block โ€” the `bridge.py` `_ensure_emoji_prefix` fallback only applies to the retired ZeroClaw path. See [protocols.md](./protocols.md#emotion-protocol) for the enforcement layers. +**Wire consequence**: the text reaching emotion dispatch must begin with an allowed emoji. On PiVoiceLLM, the per-turn suffix requests one and `_enforce_leading_emoji()` guarantees an allowed prefix, using neutral `๐Ÿ˜` when the model omits it. Persona files and `.config.yaml`'s system prompt are not forwarded on this path. See [protocols.md](./protocols.md#emotion-protocol). **Note โ€” we don't use SenseVoice's built-in SER.** The model card advertises speech emotion recognition and audio-event detection (bgm / applause / laughter / crying / coughing / sneezing). xiaozhi-server's FunASR provider returns only the transcription text; the SER/AED fields aren't piped through. That's a genuine latent capability โ€” see [latent-capabilities.md](./latent-capabilities.md#voice-pipeline-unused). diff --git a/dotty-behaviour/config.py b/dotty-behaviour/config.py index 2087090..c66aeec 100644 --- a/dotty-behaviour/config.py +++ b/dotty-behaviour/config.py @@ -168,7 +168,9 @@ def _env_float(name: str, default: float) -> float: # image_url content block. Defaults mirror bridge.py so an existing # OpenRouter key continues to work. # --------------------------------------------------------------------------- -VISION_MODEL: str = os.environ.get("VISION_MODEL", "google/gemini-2.0-flash-001") +VISION_MODEL: str = os.environ.get( + "VISION_MODEL", "google/gemini-3.1-flash-lite" +) VISION_API_URL: str = os.environ.get( "VISION_API_URL", "https://openrouter.ai/api/v1/chat/completions" ) diff --git a/dotty-behaviour/tests/test_config.py b/dotty-behaviour/tests/test_config.py new file mode 100644 index 0000000..979460f --- /dev/null +++ b/dotty-behaviour/tests/test_config.py @@ -0,0 +1,38 @@ +"""Configuration defaults and environment override behaviour.""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys + + +def _read_vlm_models(*, env: dict[str, str] | None = None) -> tuple[str, str]: + process_env = os.environ.copy() + process_env.pop("VISION_MODEL", None) + process_env.pop("VLM_MODEL", None) + if env: + process_env.update(env) + + result = subprocess.run( + [ + sys.executable, + "-c", + "import config; print(config.VISION_MODEL); print(config.VLM_MODEL)", + ], + cwd=Path(__file__).resolve().parents[1], + env=process_env, + check=True, + capture_output=True, + text=True, + ) + vision_model, vlm_model = result.stdout.splitlines() + return vision_model, vlm_model + + +def test_vision_defaults_to_live_low_latency_multimodal_model() -> None: + assert _read_vlm_models() == ( + "google/gemini-3.1-flash-lite", + "google/gemini-3.1-flash-lite", + ) diff --git a/dotty-behaviour/tests/test_routes_vision.py b/dotty-behaviour/tests/test_routes_vision.py index 29b3dab..4caf5f5 100644 --- a/dotty-behaviour/tests/test_routes_vision.py +++ b/dotty-behaviour/tests/test_routes_vision.py @@ -7,11 +7,26 @@ from dataclasses import dataclass, field from typing import Any +import pytest from fastapi.testclient import TestClient from main import app +@pytest.fixture(autouse=True) +def _disable_proactive_greeter(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep route tests from dispatching real narrative-LLM requests. + + A room-view identity match broadcasts ``face_recognized``. The app + lifespan's proactive greeter consumes that event and otherwise starts a + blocking ``requests`` call in asyncio's default executor. Cancelling the + consumer at TestClient shutdown cannot stop that worker, so the test waits + for the production 90-second timeout. Greeter behaviour is covered with + fakes in test_greeter.py; this module only owns vision-route orchestration. + """ + monkeypatch.setenv("GREETER_ENABLED", "false") + + @dataclass class _FakeVLM: calls: list[dict[str, Any]] = field(default_factory=list) diff --git a/firmware b/firmware index 35f701a..969c2b2 160000 --- a/firmware +++ b/firmware @@ -1 +1 @@ -Subproject commit 35f701a54de167b9fa69c241cfa44b070a06e627 +Subproject commit 969c2b25b2887ddaac786e6c9164db18a9d10424 diff --git a/receiveAudioHandle.py b/receiveAudioHandle.py index e24cf4c..7747b43 100644 --- a/receiveAudioHandle.py +++ b/receiveAudioHandle.py @@ -90,6 +90,10 @@ def _write_smart_mode_state(enabled: bool) -> None: _ASR_CORRECTIONS: dict[str, str] = { "doty": "Dotty", "dottie": "Dotty", + # Close phonetic substitutions observed in the 2026-07-11 filmed UAT. + # Keep this list conservative: broader variants such as Donny, Jody/Jodi, + # and Claudia are real names and must not be rewritten globally. + "duddy": "Dotty", "dotie": "Dotty", "dotti": "Dotty", "dody": "Dotty", @@ -273,7 +277,8 @@ def _is_help_request(text: str) -> bool: async def _send_led_color(conn: "ConnectionHandler", r: int, g: int, b: int) -> None: try: await _mcp_call_tool( - conn, "self.robot.set_led_color", {"r": r, "g": g, "b": b}, + conn, "self.robot.set_led_color", + {"red": r, "green": g, "blue": b}, ) except Exception: pass @@ -301,7 +306,7 @@ async def _send_led_multi( try: await _mcp_call_tool( conn, "self.robot.set_led_multi", - {"index": index, "r": r, "g": g, "b": b}, + {"index": index, "red": r, "green": g, "blue": b}, ) except Exception as exc: # The firmware may simply not support set_led_multi yet (old @@ -332,6 +337,10 @@ async def _send_set_state(conn: "ConnectionHandler", state: str) -> None: idle / talk / story_time / security / sleep / dance. The firmware StateManager handles the transition (pip update + idle profile + state_changed event back to the bridge).""" + # Record intent before the first await. Dance cleanup consults this value + # so a cancelled/finishing choreography cannot restore IDLE over a newer + # sleep/security/dance request while its MCP send is in flight. + conn._dotty_desired_state = state try: await _mcp_call_tool(conn, "self.robot.set_state", {"state": state}) except Exception as exc: @@ -663,6 +672,13 @@ async def _handle_dance(conn: "ConnectionHandler", dance_name: str) -> None: conn.logger.bind(tag=TAG).info(f"Dance mode: {dance_name}") + # Enter the firmware's sticky dance state before starting the server-side + # song timeline, making the state/event contract truthful for downstream + # consumers. Servo ownership during DANCE is a firmware responsibility. + dance_generation = getattr(conn, "_dotty_dance_generation", 0) + 1 + conn._dotty_dance_generation = dance_generation + await _send_set_state(conn, "dance") + await conn.websocket.send(json.dumps({ "type": "llm", "text": "\U0001f606", @@ -696,22 +712,13 @@ async def _handle_dance(conn: "ConnectionHandler", dance_name: str) -> None: timeline = resolve_timeline(dance) dance_task = asyncio.create_task( - execute_choreography( - conn, timeline, _send_head_angles, _send_led_color, + _run_owned_dance_choreography( + conn, dance_generation, timeline, execute_choreography, audio_latency_offset_ms=audio_offset, ) ) conn._dance_task = dance_task - def _on_dance_done(task): - async def _cleanup(): - if task.cancelled(): - await _send_head_angles(conn, 0, 0, 200) - await _send_led_color(conn, 0, 0, 0) - asyncio.ensure_future(_cleanup()) - - dance_task.add_done_callback(_on_dance_done) - if has_audio and opus_packets is not None: # Direct send: bypass tts_audio_queue and the rate controller. The # consumer's future.result(timeout=tts_timeout) trips on a 28-second @@ -735,6 +742,69 @@ async def _cleanup(): ) +async def _run_owned_dance_choreography( + conn: "ConnectionHandler", + generation: int, + timeline: list[tuple[int, str, dict]], + execute_choreography, + *, + audio_latency_offset_ms: int, +) -> None: + """Run and clean up one dance without overwriting a successor state. + + Cleanup belongs to this task (rather than a detached done-callback), so + callers can await cancellation and know all owned cleanup has finished. + The generation prevents an older dance cleaning up a replacement dance; + desired-state tracking protects security, sleep, and admin changes. + """ + cancelled = False + try: + await execute_choreography( + conn, timeline, _send_head_angles, _send_led_color, + audio_latency_offset_ms=audio_latency_offset_ms, + ) + except asyncio.CancelledError: + cancelled = True + finally: + owns_generation = ( + getattr(conn, "_dotty_dance_generation", None) == generation + ) + desired_state = getattr(conn, "_dotty_desired_state", "dance") + # Firmware's state_changed echo is asynchronous and may still report + # IDLE when a very short choreography finishes. Desired state is set + # synchronously before every local MCP request and is also refreshed + # by state_changed events (including dashboard/admin transitions). + still_dancing = desired_state == "dance" + if owns_generation and still_dancing: + if cancelled: + await _send_head_angles(conn, 0, 0, 200) + await _send_led_color(conn, 0, 0, 0) + await _send_set_state(conn, "idle") + if cancelled: + raise asyncio.CancelledError + + +async def _cancel_active_dance(conn: "ConnectionHandler", dance_task) -> None: + """Cancel and fully clean up a dance before processing its successor. + + Invalidate ownership before yielding so the task's ``finally`` cannot + race an IDLE write with a later security/sleep request. The call site + awaits this function before abort handling and successor intent parsing. + """ + conn._dotty_dance_generation = ( + getattr(conn, "_dotty_dance_generation", 0) + 1 + ) + conn._dotty_desired_state = "idle" + dance_task.cancel() + try: + await dance_task + except asyncio.CancelledError: + pass + await _send_head_angles(conn, 0, 0, 200) + await _send_led_color(conn, 0, 0, 0) + await _send_set_state(conn, "idle") + + _MIDI_RENDER_CACHE: dict[tuple, list[bytes]] = {} FLUID_SOUNDFONT = "/usr/share/sounds/sf2/FluidR3_GM.sf2" @@ -1018,7 +1088,7 @@ async def startToChat(conn: "ConnectionHandler", text): if conn.client_is_speaking and conn.client_listen_mode != "manual": dance_task = getattr(conn, "_dance_task", None) if dance_task and not dance_task.done(): - dance_task.cancel() + await _cancel_active_dance(conn, dance_task) await handleAbortMessage(conn) intent_handled = await handle_user_intent(conn, actual_text) diff --git a/scripts/uat-slice.py b/scripts/uat-slice.py index bce3c88..28386e3 100644 --- a/scripts/uat-slice.py +++ b/scripts/uat-slice.py @@ -122,8 +122,10 @@ def main() -> int: print(f"SKIP {check_id}: {exc}") failures += 1 continue - if end_wall <= start_wall: - print(f"SKIP {check_id}: end {row['end']} not after start {row['start']}") + # Equal timestamps represent an instantaneous event marker. The normal + # pre/post-roll padding turns those into useful, centred clips. + if end_wall < start_wall: + print(f"SKIP {check_id}: end {row['end']} before start {row['start']}") failures += 1 continue diff --git a/tests/test_asr_name_corrections.py b/tests/test_asr_name_corrections.py new file mode 100644 index 0000000..2d358f9 --- /dev/null +++ b/tests/test_asr_name_corrections.py @@ -0,0 +1,101 @@ +"""Regression tests for wake-name corrections on the live ASR text path.""" + +import importlib.util +import pathlib +import sys +import types +import unittest +from contextlib import contextmanager + + +_ROOT = pathlib.Path(__file__).parent.parent + + +def _stub_module(name: str, **attrs) -> None: + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + sys.modules[name] = module + + +@contextmanager +def _container_import_stubs(): + """Install container-only imports for one module load, then restore them.""" + names = ( + "core", + "core.utils", + "core.handle", + "core.utils.util", + "core.handle.abortHandle", + "core.handle.intentHandler", + "core.utils.output_counter", + "core.handle.sendAudioHandle", + "core.utils.device_command", + ) + missing = object() + previous = {name: sys.modules.get(name, missing) for name in names} + try: + for package in ("core", "core.utils", "core.handle"): + _stub_module(package) + _stub_module("core.utils.util", audio_to_data=lambda *_args, **_kwargs: None) + _stub_module("core.handle.abortHandle", handleAbortMessage=lambda *_args: None) + _stub_module("core.handle.intentHandler", handle_user_intent=lambda *_args: None) + _stub_module( + "core.utils.output_counter", + check_device_output_limit=lambda *_args: False, + ) + _stub_module( + "core.handle.sendAudioHandle", + send_stt_message=lambda *_args: None, + SentenceType=object, + ) + _stub_module("core.utils.device_command", call_tool=lambda *_args, **_kwargs: None) + yield + finally: + for name, module in previous.items(): + if module is missing: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + +# receiveAudioHandle.py is a bind-mounted upstream override, so its normal +# imports only exist inside xiaozhi-server. Provide the smallest import surface +# needed to exercise its pure text helpers on the workstation without poisoning +# sys.modules for test modules collected later. +with _container_import_stubs(): + _spec = importlib.util.spec_from_file_location( + "receive_audio_name_corrections_under_test", _ROOT / "receiveAudioHandle.py" + ) + assert _spec is not None and _spec.loader is not None + _module = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(_module) + + +class TestAsrNameCorrections(unittest.TestCase): + def test_observed_close_phonetic_variants_are_normalized(self): + for heard in ("Dottie", "Duddy"): + with self.subTest(heard=heard): + corrected = _module._apply_asr_corrections( + f"Good night, {heard}." + ) + self.assertEqual(corrected, "Good night, Dotty.") + corrected = _module._apply_phrase_corrections(corrected) + self.assertEqual( + _module._detect_state_phrase(corrected), + ("sleep", "Goodnight! ๐Ÿ˜ด"), + ) + + def test_ambiguous_real_names_are_not_rewritten(self): + for name in ("Donny", "Jody", "Jodi", "Claudia"): + with self.subTest(name=name): + text = f"Please say hello to {name}." + self.assertEqual(_module._apply_asr_corrections(text), text) + + def test_alias_matching_is_word_bounded(self): + text = "Duddybrook is a place." + self.assertEqual(_module._apply_asr_corrections(text), text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bridge_routes.py b/tests/test_bridge_routes.py index 70e4f0c..75f4527 100644 --- a/tests/test_bridge_routes.py +++ b/tests/test_bridge_routes.py @@ -165,6 +165,28 @@ def json(self): result = bridge_app._dashboard_perception_state_getter() self.assertEqual(result, {"dev-1": {"face_present": True}}) + def test_dashboard_state_getter_uses_behaviour_current_state(self): + class FakeResp: + def raise_for_status(self): + return None + + def json(self): + return {"dev-1": {"current_state": "security"}} + + self._patch_get(FakeResp()) + self.assertEqual(bridge_app._dashboard_state_getter(), "security") + + def test_dashboard_state_getter_falls_back_to_idle(self): + class FakeResp: + def raise_for_status(self): + return None + + def json(self): + return {"dev-1": {"face_present": True}} + + self._patch_get(FakeResp()) + self.assertEqual(bridge_app._dashboard_state_getter(), "idle") + def test_recent_getter_returns_fetched_list(self): class FakeResp: def raise_for_status(self): diff --git a/tests/test_dance_state_lifecycle.py b/tests/test_dance_state_lifecycle.py new file mode 100644 index 0000000..2b03391 --- /dev/null +++ b/tests/test_dance_state_lifecycle.py @@ -0,0 +1,281 @@ +"""Regression tests for the server-driven dance state lifecycle.""" + +import asyncio +import importlib.util +import pathlib +import sys +import types +import unittest +from contextlib import contextmanager +from unittest.mock import AsyncMock, patch + + +_ROOT = pathlib.Path(__file__).parent.parent + + +def _stub_module(name: str, **attrs) -> None: + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + sys.modules[name] = module + + +async def _execute_choreography(*_args, **_kwargs): + return None + + +@contextmanager +def _container_import_stubs(): + names = ( + "core", + "core.utils", + "core.handle", + "core.utils.util", + "core.handle.abortHandle", + "core.handle.intentHandler", + "core.utils.output_counter", + "core.handle.sendAudioHandle", + "core.utils.device_command", + "core.handle.dances", + ) + missing = object() + previous = {name: sys.modules.get(name, missing) for name in names} + try: + for package in ("core", "core.utils", "core.handle"): + _stub_module(package) + _stub_module("core.utils.util", audio_to_data=lambda *_args, **_kwargs: None) + _stub_module("core.handle.abortHandle", handleAbortMessage=lambda *_args: None) + _stub_module("core.handle.intentHandler", handle_user_intent=lambda *_args: None) + _stub_module("core.utils.output_counter", check_device_output_limit=lambda *_args: False) + _stub_module( + "core.handle.sendAudioHandle", + send_stt_message=lambda *_args: None, + SentenceType=object, + ) + _stub_module("core.utils.device_command", call_tool=lambda *_args, **_kwargs: None) + _stub_module( + "core.handle.dances", + DANCE_REGISTRY={ + "test": { + "intro": "Test dance", + "choreography": "test", + "duration_ms": 0, + } + }, + AUDIO_LATENCY_OFFSET_MS=0, + resolve_timeline=lambda _dance: [], + execute_choreography=_execute_choreography, + ) + yield + finally: + for name, module in previous.items(): + if module is missing: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + +with _container_import_stubs(): + _spec = importlib.util.spec_from_file_location( + "receive_audio_dance_state_under_test", _ROOT / "receiveAudioHandle.py" + ) + assert _spec is not None and _spec.loader is not None + _module = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(_module) + + +class _Logger: + def bind(self, **_kwargs): + return self + + def info(self, *_args, **_kwargs): + pass + + +class _WebSocket: + async def send(self, _message): + pass + + +class _Conn: + logger = _Logger() + websocket = _WebSocket() + session_id = "session" + sample_rate = 24000 + executor = types.SimpleNamespace(submit=lambda *_args, **_kwargs: None) + + def chat(self, _text): + pass + + +class TestDanceStateLifecycle(unittest.TestCase): + def test_led_helper_uses_firmware_schema_names(self): + async def run(): + call = AsyncMock() + with patch.object(_module, "_mcp_call_tool", new=call): + await _module._send_led_color(_Conn(), 1, 2, 3) + call.assert_awaited_once_with( + unittest.mock.ANY, + "self.robot.set_led_color", + {"red": 1, "green": 2, "blue": 3}, + ) + + asyncio.run(run()) + + def test_dance_enters_firmware_state_then_returns_to_idle(self): + async def run(): + states = [] + + async def record_state(_conn, state): + states.append(state) + + with ( + _container_import_stubs(), + patch.object(_module, "_send_set_state", side_effect=record_state), + patch.object(_module, "_send_led_color", new=AsyncMock()), + ): + await _module._handle_dance(_Conn(), "test") + # Let the completed choreography task and its cleanup run. + await asyncio.sleep(0.01) + + self.assertEqual(states, ["dance", "idle"]) + + asyncio.run(run()) + + def test_fast_dance_cleans_up_before_state_event_echo_arrives(self): + async def run(): + conn = _Conn() + conn._dotty_dance_generation = 1 + conn._dotty_desired_state = "dance" + conn.current_state = "idle" # production-shaped stale event echo + states = [] + + async def record_state(target_conn, state): + target_conn._dotty_desired_state = state + states.append(state) + + with ( + patch.object(_module, "_send_set_state", side_effect=record_state), + patch.object(_module, "_send_led_color", new=AsyncMock()), + ): + await _module._run_owned_dance_choreography( + conn, 1, [], _execute_choreography, + audio_latency_offset_ms=0, + ) + + self.assertEqual(states, ["idle"]) + + asyncio.run(run()) + + def test_cancelled_dance_does_not_overwrite_new_security_or_sleep_state(self): + async def run(successor): + conn = _Conn() + conn._dotty_dance_generation = 1 + conn._dotty_desired_state = "dance" + conn.current_state = "dance" + started = asyncio.Event() + + async def blocked_choreography(*_args, **_kwargs): + started.set() + await asyncio.Event().wait() + + set_state = AsyncMock() + set_head = AsyncMock() + set_led = AsyncMock() + with ( + patch.object(_module, "_send_set_state", new=set_state), + patch.object(_module, "_send_head_angles", new=set_head), + patch.object(_module, "_send_led_color", new=set_led), + ): + task = asyncio.create_task( + _module._run_owned_dance_choreography( + conn, 1, [], blocked_choreography, + audio_latency_offset_ms=0, + ) + ) + await started.wait() + # Mirrors _send_set_state's synchronous-before-await intent + # update plus the firmware state_changed observation. + conn._dotty_desired_state = successor + conn.current_state = successor + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + + set_state.assert_not_awaited() + set_head.assert_not_awaited() + set_led.assert_not_awaited() + + for successor in ("security", "sleep"): + with self.subTest(successor=successor): + asyncio.run(run(successor)) + + def test_replaced_dance_owns_cleanup(self): + async def run(): + conn = _Conn() + conn._dotty_dance_generation = 2 + conn._dotty_desired_state = "dance" + conn.current_state = "dance" + set_state = AsyncMock() + set_head = AsyncMock() + set_led = AsyncMock() + with ( + patch.object(_module, "_send_set_state", new=set_state), + patch.object(_module, "_send_head_angles", new=set_head), + patch.object(_module, "_send_led_color", new=set_led), + ): + await _module._run_owned_dance_choreography( + conn, 1, [], _execute_choreography, + audio_latency_offset_ms=0, + ) + + set_state.assert_not_awaited() + set_head.assert_not_awaited() + set_led.assert_not_awaited() + + asyncio.run(run()) + + def test_call_site_finishes_cancel_cleanup_before_later_successor_state(self): + async def run(successor): + conn = _Conn() + conn._dotty_dance_generation = 1 + conn._dotty_desired_state = "dance" + conn.current_state = "dance" + started = asyncio.Event() + states = [] + + async def blocked_choreography(*_args, **_kwargs): + started.set() + await asyncio.Event().wait() + + async def record_state(target_conn, state): + target_conn._dotty_desired_state = state + states.append(state) + + with ( + patch.object(_module, "_send_set_state", side_effect=record_state), + patch.object(_module, "_send_head_angles", new=AsyncMock()), + patch.object(_module, "_send_led_color", new=AsyncMock()), + ): + dance_task = asyncio.create_task( + _module._run_owned_dance_choreography( + conn, 1, [], blocked_choreography, + audio_latency_offset_ms=0, + ) + ) + await started.wait() + # Actual startToChat ordering: cancel dance, await abort work, + # then parse and dispatch the successor state phrase. + await _module._cancel_active_dance(conn, dance_task) + await asyncio.sleep(0) # handleAbortMessage yields here + await _module._send_set_state(conn, successor) + + self.assertEqual(states, ["idle", successor]) + + for successor in ("security", "sleep", "dance"): + with self.subTest(successor=successor): + asyncio.run(run(successor)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_device_command.py b/tests/test_device_command.py index 549934b..07007e9 100644 --- a/tests/test_device_command.py +++ b/tests/test_device_command.py @@ -205,7 +205,9 @@ async def go(): type(self).active["dev-1"] = conn srv = self._server() r1 = await srv._dotty_set_state(self._request({"state": "sleep"})) + self.assertEqual(conn._dotty_desired_state, "sleep") r2 = await srv._dotty_set_state(self._request({"state": "idle"})) + self.assertEqual(conn._dotty_desired_state, "idle") self.assertEqual((r1.status, r2.status), (200, 200)) # Sends are _spawn()-ed fire-and-forget; let the tasks run. await asyncio.sleep(0.02) diff --git a/tests/test_uat_slice.py b/tests/test_uat_slice.py new file mode 100644 index 0000000..5a932be --- /dev/null +++ b/tests/test_uat_slice.py @@ -0,0 +1,90 @@ +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "uat-slice.py" + + +def load_uat_slice(): + spec = importlib.util.spec_from_file_location("uat_slice", SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def run_slicer(tmp_path, monkeypatch, capsys, row): + module = load_uat_slice() + results = tmp_path / "2026-07-11" / "results.csv" + results.parent.mkdir() + results.write_text( + "check_id,verdict,source,start,end,note\n" + row + "\n", + encoding="utf-8", + ) + video = tmp_path / "phone.mov" + video.touch() + commands = [] + + def fake_run(command): + commands.append(command) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(module.subprocess, "run", fake_run) + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + str(results), + "--video", + f"phone={video}", + "--sync", + "phone=00:00:00@19:41:32", + "--out", + str(tmp_path / "clips"), + ], + ) + return module.main(), commands, capsys.readouterr().out + + +def test_equal_timestamps_create_pad_only_event_clip(tmp_path, monkeypatch, capsys): + result, commands, output = run_slicer( + tmp_path, + monkeypatch, + capsys, + "UT7,FAIL,phone,20:01:31,20:01:31,event marker", + ) + + assert result == 0 + assert len(commands) == 1 + assert commands[0][commands[0].index("-ss") + 1] == "00:19:56.000" + assert commands[0][commands[0].index("-to") + 1] == "00:20:02.000" + assert "FAIL UT7" in output + + +def test_blank_timestamps_remain_skipped(tmp_path, monkeypatch, capsys): + result, commands, output = run_slicer( + tmp_path, + monkeypatch, + capsys, + "US7,FAIL,phone,,,missing observation time", + ) + + assert result == 1 + assert commands == [] + assert "SKIP US7" in output + + +def test_reversed_timestamps_remain_invalid(tmp_path, monkeypatch, capsys): + result, commands, output = run_slicer( + tmp_path, + monkeypatch, + capsys, + "UC1,PASS,phone,19:45:15,19:45:06,reversed window", + ) + + assert result == 1 + assert commands == [] + assert "end 19:45:06 before start 19:45:15" in output diff --git a/tests/test_voice_content_filter.py b/tests/test_voice_content_filter.py index be1b1ea..28ad5d9 100644 --- a/tests/test_voice_content_filter.py +++ b/tests/test_voice_content_filter.py @@ -14,6 +14,7 @@ import importlib.util as _ilu import os import sys +import tempfile import types import unittest from pathlib import Path @@ -176,10 +177,18 @@ def close(self): pass def _respond(self, chunks, kid_mode): - env = {"DOTTY_KID_MODE": "true" if kid_mode else "false"} - with patch.dict(os.environ, env): - provider = self.LLMProvider({}, client=self._FakeClient(chunks)) - return list(provider.response("s", [{"role": "user", "content": "hi"}])) + with tempfile.TemporaryDirectory() as tmp: + env = { + "DOTTY_KID_MODE": "true" if kid_mode else "false", + "DOTTY_KID_MODE_STATE": str(Path(tmp) / "missing-kid-mode"), + } + # PiVoice refreshes kid-mode at response time, so keep the + # environment patched through both construction and the turn. + with patch.dict(os.environ, env): + provider = self.LLMProvider({}, client=self._FakeClient(chunks)) + return list(provider.response( + "s", [{"role": "user", "content": "hi"}], + )) def test_kid_on_blocked_turn_replaced(self): out = self._respond(["๐Ÿ˜Š Sure, coc", "aine is a stimulant. ", "It acts on..."], True) @@ -217,19 +226,24 @@ def close(self): pass client = DrainingClient() - with patch.dict(os.environ, {"DOTTY_KID_MODE": "true"}): - provider = self.LLMProvider({}, client=client) - - self.assertEqual( - list(provider.response("s", [{"role": "user", "content": "first"}])), - [REPLACEMENT], - ) - self.assertEqual(client.completed, 1) - self.assertEqual( - list(provider.response("s", [{"role": "user", "content": "second"}])), - ["๐Ÿ˜Š Clean next turn."], - ) - self.assertEqual(client.completed, 2) + with tempfile.TemporaryDirectory() as tmp: + env = { + "DOTTY_KID_MODE": "true", + "DOTTY_KID_MODE_STATE": str(Path(tmp) / "missing-kid-mode"), + } + with patch.dict(os.environ, env): + provider = self.LLMProvider({}, client=client) + + self.assertEqual( + list(provider.response("s", [{"role": "user", "content": "first"}])), + [REPLACEMENT], + ) + self.assertEqual(client.completed, 1) + self.assertEqual( + list(provider.response("s", [{"role": "user", "content": "second"}])), + ["๐Ÿ˜Š Clean next turn."], + ) + self.assertEqual(client.completed, 2) class TestOpenAICompatWiring(unittest.TestCase): From 5d0651dd459d5256b31fcfa9d269b2e89cf702d3 Mon Sep 17 00:00:00 2001 From: Brett Kinny Date: Sat, 11 Jul 2026 23:53:36 +1000 Subject: [PATCH 2/2] fix(ota): advertise device-reachable download host --- custom-providers/xiaozhi-patches/ota_handler.py | 14 +++++++++++++- tests/test_ota_version.py | 9 +++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/custom-providers/xiaozhi-patches/ota_handler.py b/custom-providers/xiaozhi-patches/ota_handler.py index d1896c8..63af673 100644 --- a/custom-providers/xiaozhi-patches/ota_handler.py +++ b/custom-providers/xiaozhi-patches/ota_handler.py @@ -7,6 +7,7 @@ import re import glob from typing import Dict, List, Tuple +from urllib.parse import urlparse from aiohttp import web from core.auth import AuthManager @@ -16,6 +17,16 @@ TAG = __name__ +def _download_host(config: dict, fallback: str) -> str: + """Return the device-reachable host configured for WebSocket traffic.""" + websocket_url = config.get("server", {}).get("websocket", "") + if websocket_url and "ไฝ ็š„" not in websocket_url: + hostname = urlparse(websocket_url).hostname + if hostname: + return hostname + return fallback + + def _safe_basename(filename: str) -> str: # Prevent directory traversal return os.path.basename(filename) @@ -335,8 +346,9 @@ async def handle_post(self, request): # device gets handed back the vision endpoint as a download # URL, returning 405. chosen_version = ver + download_host = _download_host(self.config, local_ip) chosen_url = ( - f"http://{local_ip}:{http_port}/xiaozhi/ota/download/{fname}" + f"http://{download_host}:{http_port}/xiaozhi/ota/download/{fname}" ) break diff --git a/tests/test_ota_version.py b/tests/test_ota_version.py index 2464239..11cf19d 100644 --- a/tests/test_ota_version.py +++ b/tests/test_ota_version.py @@ -47,10 +47,19 @@ def __init__(self, *a, **k): _is_higher = _mod._is_higher_version _key = _mod._parse_version +_download_host = _mod._download_host class TestOtaVersion(unittest.TestCase): + def test_download_host_uses_configured_websocket_host(self): + config = {"server": {"websocket": "ws://192.168.1.67:8000/xiaozhi/v1/"}} + self.assertEqual(_download_host(config, "172.21.0.2"), "192.168.1.67") + + def test_download_host_falls_back_for_placeholder(self): + config = {"server": {"websocket": "ws://ไฝ ็š„ๅฑ€ๅŸŸ็ฝ‘IP:8000/xiaozhi/v1/"}} + self.assertEqual(_download_host(config, "172.21.0.2"), "172.21.0.2") + # The headline regression: a pre-release must NOT outrank its GA release. def test_prerelease_not_higher_than_release(self): self.assertFalse(_is_higher("1.2.3-rc1", "1.2.3"))