From 3e7973b0a25a90167cd63c57ac3fd784464c703b Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Thu, 22 Oct 2026 22:36:58 +0800 Subject: [PATCH 1/8] feat(twilio): inbound ConversationRelay and a response watchdog Inbound calls for agents listed in ConversationRelay.AgentIds now connect to ConversationRelay (streamed replies) instead of the Gather flow. The welcome prompts queued by session hooks become the relay greeting through their Twilio:PromptTexts text. Calls that arrive with an intent keep the Gather flow, which already has their first message queued. ConversationRelay.ResponseWatchdogSeconds (off by default) says one short filler from ResponseWatchdogPrompts (default hold-on-1..5) when the agent has said nothing that long after the caller finished, at most once per turn. ResponseWatchdogAgentIds limits it to some agents. The filler is decided while holding the send lock, so it never lands inside a streamed reply, and a turn whose reply is already in never gets one. Co-Authored-By: Claude Opus 5.5 --- .../Controllers/TwilioInboundController.cs | 14 +++ .../Services/ConversationRelaySession.cs | 85 +++++++++++++++++-- .../Services/TwilioService.cs | 9 ++ .../Settings/ConversationRelaySetting.cs | 24 +++++- 4 files changed, 126 insertions(+), 6 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index 7c1f083e4..8fab8aee1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -93,6 +93,20 @@ await HookEmitter.Emit(_services, { response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent); } + // A call that arrives with an intent already has its first message queued for the Gather flow + else if (string.IsNullOrWhiteSpace(request.Intent) && _settings.ConversationRelay.IsEnabledFor(request.AgentId)) + { + // ConversationRelay speaks text only: the prompts the session hooks queued become the greeting + var greeting = string.Join(" ", instruction.SpeechPaths + .Select(twilio.GetPromptText) + .Where(text => !string.IsNullOrWhiteSpace(text))); + if (greeting.Length == 0 && instruction.SpeechPaths.Count > 0) + { + _logger.LogWarning($"No prompt text for the welcome speech of agent {request.AgentId}; " + + "starting ConversationRelay without a greeting."); + } + response = twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, greeting); + } else { if (string.IsNullOrWhiteSpace(request.Intent)) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 80ae82ec8..47c222ccd 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -15,7 +15,9 @@ public class ConversationRelaySession { private const string DefaultSayItAgainText = "Sorry, can you say it again?"; private const string DefaultErrorText = "Sorry, something went wrong. Please call back later."; + private const string DefaultHoldOnText = "One moment, please."; private const double SpokenCharsPerSecond = 15; + private static readonly List DefaultWatchdogPrompts = ["hold-on-1", "hold-on-2", "hold-on-3", "hold-on-4", "hold-on-5"]; private readonly WebSocket _socket; private readonly string _agentId; @@ -48,6 +50,9 @@ public class ConversationRelaySession private bool _streamedThisTurn; private bool _dropStream; + // Anything was said to the caller during the current turn; the response watchdog stays quiet once it is set + private bool _spokeThisTurn; + private string? _callSid; private string? _from; private string? _to; @@ -296,8 +301,14 @@ private async Task ProcessPendingAsync() _utteranceOpen = false; _streamedThisTurn = false; _dropStream = false; + _spokeThisTurn = false; } + using var watchdogCts = new CancellationTokenSource(); + var watchdog = _settings.ConversationRelay.IsResponseWatchdogEnabledFor(_agentId) + ? WatchResponseAsync(message.SeqNumber, watchdogCts.Token) + : Task.CompletedTask; + var stream = _settings.ConversationRelay.StreamReplies; var callbacks = new AgentTurnCallbacks { @@ -332,6 +343,16 @@ private async Task ProcessPendingAsync() await FailAsync(); return; } + finally + { + // The reply is in: a filler that already passed its delay must not slip in ahead of it + lock (_sync) + { + _spokeThisTurn = true; + } + watchdogCts.Cancel(); + await SwallowAsync(watchdog); + } await CloseUtteranceAsync(); if (reply == null @@ -354,6 +375,47 @@ private async Task ProcessPendingAsync() } } + /// + /// Says a short filler when the agent has said nothing this turn after ResponseWatchdogSeconds, + /// so the caller does not sit in silence while the model thinks or a tool runs. + /// + private async Task WatchResponseAsync(int seqNumber, CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(_settings.ConversationRelay.ResponseWatchdogSeconds), cancellationToken); + + var text = PickFillerText(); + await _sendLock.WaitAsync(cancellationToken); + try + { + // Decided while holding the send lock, so a reply cannot start streaming between the check and the + // filler; a stream that already began has set _utteranceOpen and is waiting for this lock + lock (_sync) + { + if (_ended || _seqNum != seqNumber || _spokeThisTurn || _utteranceOpen) + { + return; + } + _spokeThisTurn = true; + } + await SendLockedAsync(new { type = "text", token = text, last = true }); + } + finally + { + _sendLock.Release(); + } + MarkSpeaking(EstimateSpeech(text.Length)); + } + + private string PickFillerText() + { + var names = _settings.ConversationRelay.ResponseWatchdogPrompts; + var texts = (names.Count > 0 ? names : DefaultWatchdogPrompts) + .Select(name => _settings.PromptTexts.TryGetValue(name, out var text) ? text : null) + .Where(text => !string.IsNullOrWhiteSpace(text)) + .ToList(); + return texts.Count == 0 ? DefaultHoldOnText : texts[Random.Shared.Next(texts.Count)]!; + } + private async Task WatchIdleAsync(CancellationToken cancellationToken) { var relay = _settings.ConversationRelay; @@ -426,6 +488,7 @@ private async Task StreamTextAsync(string text) _streamSegment.Append(text); _utteranceOpen = true; _streamedThisTurn = true; + _spokeThisTurn = true; } await SendAsync(new { type = "text", token = text, last = false }); @@ -543,6 +606,7 @@ private async Task SpeakAsync(string? text) { return; } + _spokeThisTurn = true; } await SendAsync(new { type = "text", token = text, last = true }); @@ -587,9 +651,24 @@ private record PendingInput(string Content, string? Digits); private async Task SendAsync(object payload) { - var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload)); await _sendLock.WaitAsync(); try + { + await SendLockedAsync(payload); + } + finally + { + _sendLock.Release(); + } + } + + /// + /// Sends with already held by the caller. + /// + private async Task SendLockedAsync(object payload) + { + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload)); + try { if (_socket.State == WebSocketState.Open) { @@ -602,10 +681,6 @@ private async Task SendAsync(object payload) // The call already moved on, e.g. a hang-up or transfer redirected it _logger.LogInformation($"ConversationRelay socket for conversation {_conversationId} is gone: {ex.Message}"); } - finally - { - _sendLock.Release(); - } } private void MarkSpeaking(TimeSpan duration) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 07c5caeac..9c1ef3e3b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -464,6 +464,15 @@ public void AppendSpeech(TwiML parent, string? conversationId, string speechPath parent.Append(CreateSay(data.ToString())); } + /// + /// The configured text of a prerecorded prompt, or null when the path has none (e.g. a per-conversation file). + /// ConversationRelay can only speak text, so this is how its greeting reuses the prompts a hook queued. + /// + public string? GetPromptText(string speechPath) + { + return TryGetPromptText(speechPath, out var text) ? text : null; + } + /// /// Looks up the configured text of a prerecorded prompt such as "twilio/say-it-again-1.mp3". /// Files generated per conversation (twilio/voice/speeches/...) never match. diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index 0fbc419d8..f9797f1d6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -9,7 +9,7 @@ public class ConversationRelaySetting public bool Enabled { get; set; } /// - /// Agents that use ConversationRelay for outbound calls. Empty means every agent. + /// Agents that use ConversationRelay for their calls, outbound and inbound. Empty means every agent. /// public List AgentIds { get; set; } = []; @@ -69,8 +69,30 @@ public class ConversationRelaySetting /// public bool ValidateSignature { get; set; } = true; + /// + /// Seconds to wait for the agent's first words after the caller finishes speaking before saying a short + /// filler (e.g. "One moment, please."), at most once per turn. 0 turns it off. + /// + public double ResponseWatchdogSeconds { get; set; } + + /// + /// Agents the response watchdog applies to. Empty means every ConversationRelay agent. + /// + public List ResponseWatchdogAgentIds { get; set; } = []; + + /// + /// Names of Twilio:PromptTexts entries the filler is picked from at random. Empty uses hold-on-1 to hold-on-5. + /// + public List ResponseWatchdogPrompts { get; set; } = []; + public bool IsEnabledFor(string? agentId) { return Enabled && (AgentIds.Count == 0 || (agentId != null && AgentIds.Contains(agentId))); } + + public bool IsResponseWatchdogEnabledFor(string? agentId) + { + return ResponseWatchdogSeconds > 0 + && (ResponseWatchdogAgentIds.Count == 0 || (agentId != null && ResponseWatchdogAgentIds.Contains(agentId))); + } } From 644ef6944554db2a975cacaea016ec420c2fbf4f Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 23 Oct 2026 07:16:44 +0800 Subject: [PATCH 2/8] feat(twilio): repeat the response watchdog filler at intervals, never the same twice ResponseWatchdogSeconds becomes ResponseWatchdogIntervals: seconds of silence before each filler while the agent works on a reply. The first counts from when the caller finished, each next one from when whatever was said last (a filler, an indication, a streamed reply) ends; the last value repeats until the reply comes. The first filler of a turn comes from ResponseWatchdogPrompts (default hold-on-1..5), later ones from ResponseWatchdogLongPrompts (default hold-on-long-1..10), and no sentence is repeated in a turn while unused ones remain. Still nothing while a reply is streaming, and nothing once the turn's reply is in. Co-Authored-By: Claude Opus 5.5 --- .../Services/ConversationRelaySession.cs | 83 ++++++++++++++----- .../Settings/ConversationRelaySetting.cs | 16 ++-- 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 47c222ccd..7e7c37092 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -1,4 +1,4 @@ -using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Models.Relay; using System.IO; using System.Net.WebSockets; @@ -18,6 +18,8 @@ public class ConversationRelaySession private const string DefaultHoldOnText = "One moment, please."; private const double SpokenCharsPerSecond = 15; private static readonly List DefaultWatchdogPrompts = ["hold-on-1", "hold-on-2", "hold-on-3", "hold-on-4", "hold-on-5"]; + private static readonly List DefaultWatchdogLongPrompts = Enumerable.Range(1, 10).Select(i => $"hold-on-long-{i}").ToList(); + private static readonly TimeSpan WatchdogPollInterval = TimeSpan.FromMilliseconds(100); private readonly WebSocket _socket; private readonly string _agentId; @@ -50,8 +52,9 @@ public class ConversationRelaySession private bool _streamedThisTurn; private bool _dropStream; - // Anything was said to the caller during the current turn; the response watchdog stays quiet once it is set - private bool _spokeThisTurn; + // The current turn's reply is in; the response watchdog says nothing more once it is set + private bool _turnReplied; + private DateTime _turnStartedUtc; private string? _callSid; private string? _from; @@ -301,7 +304,8 @@ private async Task ProcessPendingAsync() _utteranceOpen = false; _streamedThisTurn = false; _dropStream = false; - _spokeThisTurn = false; + _turnReplied = false; + _turnStartedUtc = DateTime.UtcNow; } using var watchdogCts = new CancellationTokenSource(); @@ -348,7 +352,7 @@ private async Task ProcessPendingAsync() // The reply is in: a filler that already passed its delay must not slip in ahead of it lock (_sync) { - _spokeThisTurn = true; + _turnReplied = true; } watchdogCts.Cancel(); await SwallowAsync(watchdog); @@ -376,26 +380,59 @@ private async Task ProcessPendingAsync() } /// - /// Says a short filler when the agent has said nothing this turn after ResponseWatchdogSeconds, - /// so the caller does not sit in silence while the model thinks or a tool runs. + /// While the agent works on a reply, says a filler each time the line has been silent for the next + /// ResponseWatchdogIntervals value, so the caller never sits in a long silence while the model thinks or a tool runs. + /// Silence counts from the end of whatever was said last (the caller, a filler, an indication), and no filler is + /// said twice in a turn while unused ones remain. /// private async Task WatchResponseAsync(int seqNumber, CancellationToken cancellationToken) { - await Task.Delay(TimeSpan.FromSeconds(_settings.ConversationRelay.ResponseWatchdogSeconds), cancellationToken); + var relay = _settings.ConversationRelay; + var said = new HashSet(); + for (var step = 0; ; step++) + { + var interval = TimeSpan.FromSeconds(relay.ResponseWatchdogIntervals[Math.Min(step, relay.ResponseWatchdogIntervals.Count - 1)]); + while (true) + { + await Task.Delay(WatchdogPollInterval, cancellationToken); + lock (_sync) + { + if (_ended || _seqNum != seqNumber || _turnReplied) + { + return; + } + // _quietSinceUtc is the estimated end of what was last said, or the caller's last input + var quietSince = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < _turnStartedUtc ? _turnStartedUtc : _quietSinceUtc; + if (!_utteranceOpen && DateTime.UtcNow - quietSince >= interval) + { + break; + } + } + } - var text = PickFillerText(); - await _sendLock.WaitAsync(cancellationToken); + var text = PickFillerText(step == 0 ? relay.ResponseWatchdogPrompts : relay.ResponseWatchdogLongPrompts, + step == 0 ? DefaultWatchdogPrompts : DefaultWatchdogLongPrompts, said); + if (!await SpeakFillerAsync(seqNumber, text)) + { + return; + } + said.Add(text); + } + } + + private async Task SpeakFillerAsync(int seqNumber, string text) + { + await _sendLock.WaitAsync(); try { // Decided while holding the send lock, so a reply cannot start streaming between the check and the // filler; a stream that already began has set _utteranceOpen and is waiting for this lock lock (_sync) { - if (_ended || _seqNum != seqNumber || _spokeThisTurn || _utteranceOpen) + if (_ended || _seqNum != seqNumber || _turnReplied || _utteranceOpen) { - return; + return false; } - _spokeThisTurn = true; } await SendLockedAsync(new { type = "text", token = text, last = true }); } @@ -404,16 +441,26 @@ private async Task WatchResponseAsync(int seqNumber, CancellationToken cancellat _sendLock.Release(); } MarkSpeaking(EstimateSpeech(text.Length)); + return true; } - private string PickFillerText() + private string PickFillerText(List names, List defaultNames, HashSet said) { - var names = _settings.ConversationRelay.ResponseWatchdogPrompts; - var texts = (names.Count > 0 ? names : DefaultWatchdogPrompts) + var texts = (names.Count > 0 ? names : defaultNames) .Select(name => _settings.PromptTexts.TryGetValue(name, out var text) ? text : null) .Where(text => !string.IsNullOrWhiteSpace(text)) + .Select(text => text!) + .Distinct() .ToList(); - return texts.Count == 0 ? DefaultHoldOnText : texts[Random.Shared.Next(texts.Count)]!; + if (texts.Count == 0) + { + return DefaultHoldOnText; + } + + // A different sentence each time; repeat only once every one has been said + var unsaid = texts.Where(text => !said.Contains(text)).ToList(); + var pool = unsaid.Count > 0 ? unsaid : texts; + return pool[Random.Shared.Next(pool.Count)]; } private async Task WatchIdleAsync(CancellationToken cancellationToken) @@ -488,7 +535,6 @@ private async Task StreamTextAsync(string text) _streamSegment.Append(text); _utteranceOpen = true; _streamedThisTurn = true; - _spokeThisTurn = true; } await SendAsync(new { type = "text", token = text, last = false }); @@ -606,7 +652,6 @@ private async Task SpeakAsync(string? text) { return; } - _spokeThisTurn = true; } await SendAsync(new { type = "text", token = text, last = true }); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index f9797f1d6..cb6cbbfe9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -70,10 +70,11 @@ public class ConversationRelaySetting public bool ValidateSignature { get; set; } = true; /// - /// Seconds to wait for the agent's first words after the caller finishes speaking before saying a short - /// filler (e.g. "One moment, please."), at most once per turn. 0 turns it off. + /// Seconds of silence on the line, while the agent works on its reply, before each filler such as "One moment, please.": + /// the first counts from when the caller finished, each next one from when the previous filler (or anything else + /// we said) finished. The last value repeats until the reply comes. Empty turns the watchdog off. /// - public double ResponseWatchdogSeconds { get; set; } + public List ResponseWatchdogIntervals { get; set; } = []; /// /// Agents the response watchdog applies to. Empty means every ConversationRelay agent. @@ -81,10 +82,15 @@ public class ConversationRelaySetting public List ResponseWatchdogAgentIds { get; set; } = []; /// - /// Names of Twilio:PromptTexts entries the filler is picked from at random. Empty uses hold-on-1 to hold-on-5. + /// Names of Twilio:PromptTexts entries the first filler of a turn is picked from. Empty uses hold-on-1 to hold-on-5. /// public List ResponseWatchdogPrompts { get; set; } = []; + /// + /// Names of Twilio:PromptTexts entries the later fillers of a long wait are picked from. Empty uses hold-on-long-1 to hold-on-long-10. + /// + public List ResponseWatchdogLongPrompts { get; set; } = []; + public bool IsEnabledFor(string? agentId) { return Enabled && (AgentIds.Count == 0 || (agentId != null && AgentIds.Contains(agentId))); @@ -92,7 +98,7 @@ public bool IsEnabledFor(string? agentId) public bool IsResponseWatchdogEnabledFor(string? agentId) { - return ResponseWatchdogSeconds > 0 + return ResponseWatchdogIntervals.Count > 0 && ResponseWatchdogIntervals.All(seconds => seconds > 0) && (ResponseWatchdogAgentIds.Count == 0 || (agentId != null && ResponseWatchdogAgentIds.Contains(agentId))); } } From d15a2c4d3e764aceadd9929390307d037537c1ba Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 23 Oct 2026 14:32:15 +0800 Subject: [PATCH 3/8] feat(twilio): close the streamed utterance when a response ends in a tool call A line the model says before a slow tool ("Okay, that's everything I need") stayed an open utterance until the next response started streaming, so the response watchdog never filled the silence while the tool ran. The relay now closes it as soon as the response turns out to be a tool call (AgentTurnCallbacks.OnStreamEndedWithToolCall), and the watchdog counts from the end of that line. Co-Authored-By: Claude Opus 5.5 --- .../BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs | 6 ++++++ .../Services/ConversationRelaySession.cs | 5 ++++- .../Services/TwilioConversationProcessor.cs | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs index f5735f168..fcbb782b7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs @@ -21,4 +21,10 @@ public class AgentTurnCallbacks /// Called when the model starts writing a new response, e.g. after a tool call /// public Func? OnStreamStarted { get; init; } + + /// + /// Called when a response the model wrote ends in a tool call: what it said before the call is complete, + /// and the tool may take a while before anything else is said + /// + public Func? OnStreamEndedWithToolCall { get; init; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 7e7c37092..3f54fea6c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -318,7 +318,10 @@ private async Task ProcessPendingAsync() { OnIndication = SpeakIndicationAsync, OnStreamText = stream ? StreamTextAsync : null, - OnStreamStarted = stream ? CloseUtteranceAsync : null + OnStreamStarted = stream ? CloseUtteranceAsync : null, + // A line said before a slow tool ("Okay, that's everything I need") is spoken now, and the watchdog + // counts the silence after it instead of waiting for the next response to close it + OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null }; AssistantMessage? reply; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs index 213c024c0..6ae5ce0b9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs @@ -71,6 +71,13 @@ public static class TwilioConversationProcessor await callbacks!.OnStreamText!(data.Data.Content); } }; + listeners[ChatEvent.AfterReceiveLlmStreamMessage] = async data => + { + if (data.Data.Role == AgentRole.Function && callbacks!.OnStreamEndedWithToolCall != null) + { + await callbacks.OnStreamEndedWithToolCall(); + } + }; } using var container = observer.SubscribeObservers>( From 7364c1c903b340914ab59f2bcdf4955b34f47067 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 23 Oct 2026 15:05:55 +0800 Subject: [PATCH 4/8] Log what the model wrote when an OpenAI chat reply hits the output limit A reply cut off by max_output_tokens is replaced by a fixed notice, so the log is the only record of what the model wrote. The streaming path logged nothing, and the non-streaming paths left out the tool calls, which is where a cut-off usually happens. All three now log the text and the tool calls with their arguments as far as the model got. Co-Authored-By: Claude Opus 5.5 --- .../Chat/ChatCompletionProvider.Chat.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs index f731a9842..7f91dbbe8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs @@ -56,7 +56,7 @@ private async Task InnerGetChatCompletions(Agent agent, List InnerGetChatCompletionsAsync(Agent agent, } else if (reason == ChatFinishReason.Length) { - _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}"); + _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}, ToolCalls: {DescribeToolCalls(ToLlmToolCalls(value.ToolCalls))}"); msg = new RoleDialogModel(AgentRole.Assistant, $"AI response exceeded max output length") { @@ -321,6 +321,13 @@ private async Task InnerGetChatCompletionsStreamingAsync(Agent } else if (choice.FinishReason.HasValue) { + if (choice.FinishReason == ChatFinishReason.Length) + { + // What the model wrote before the limit is replaced by the notice below, so this is the only record of it + _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsStreamingAsync)}, Reason: {choice.FinishReason}, Agent: {agent.Name}, " + + $"MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{textStream.GetText()}, ToolCalls: {DescribeToolCalls(ReconstructToolCalls(toolCalls))}"); + } + var text = choice.FinishReason == ChatFinishReason.Length ? "Model reached the maximum number of tokens allowed." : choice.FinishReason == ChatFinishReason.ContentFilter ? "Content is omitted due to content filter rule." : choice.FinishReason.Value.ToString(); @@ -759,6 +766,12 @@ private static List ToLlmToolCalls(IEnumerable? toolC .Select(x => new LlmToolCall(x.Id, x.FunctionName, x.FunctionArguments?.ToString())) .ToList(); + /// + /// The tool calls of a reply cut off by the output limit, arguments as far as the model got + /// + private static string DescribeToolCalls(List calls) + => string.Join(", ", calls.Select(x => $"{x.FunctionName}({x.FunctionArgs})")); + /// /// Rebuilds the tool calls a streaming reply asked for. /// From 1fb0499edd29ed94be1cb8ad0370c9822be650c4 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 23 Oct 2026 15:47:47 +0800 Subject: [PATCH 5/8] Let a Twilio session hook supply the ConversationRelay greeting text The inbound ConversationRelay greeting could only come from the prompt texts behind the speech paths the session hooks queue, which are fixed. A hook that writes ConversationalVoiceResponse.Text (e.g. a greeting with the caller's name) now has that said instead; without it the greeting is built as before. Co-Authored-By: Claude Opus 5.5 --- .../Controllers/TwilioInboundController.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index 8fab8aee1..d1ee441a1 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -96,10 +96,13 @@ await HookEmitter.Emit(_services, // A call that arrives with an intent already has its first message queued for the Gather flow else if (string.IsNullOrWhiteSpace(request.Intent) && _settings.ConversationRelay.IsEnabledFor(request.AgentId)) { - // ConversationRelay speaks text only: the prompts the session hooks queued become the greeting - var greeting = string.Join(" ", instruction.SpeechPaths - .Select(twilio.GetPromptText) - .Where(text => !string.IsNullOrWhiteSpace(text))); + // ConversationRelay speaks text only: a greeting a session hook wrote (e.g. with the caller's name) is + // said as it is, otherwise the prompts the session hooks queued become the greeting + var greeting = !string.IsNullOrWhiteSpace(instruction.Text) + ? instruction.Text + : string.Join(" ", instruction.SpeechPaths + .Select(twilio.GetPromptText) + .Where(text => !string.IsNullOrWhiteSpace(text))); if (greeting.Length == 0 && instruction.SpeechPaths.Count > 0) { _logger.LogWarning($"No prompt text for the welcome speech of agent {request.AgentId}; " + From 3747588b036ee819f67281195883338054bbc0da Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Fri, 23 Oct 2026 15:48:07 +0800 Subject: [PATCH 6/8] fix(twilio): recheck relay send decisions once the send lock is held A streamed piece, an idle "say it again" prompt or a spoken line was checked first and sent after waiting for the send lock, so the decision could go stale meanwhile: - a streamed piece queued behind a filler could go out after the caller cut in or the session ended, leaving one fragment of the reply; - an idle prompt could be sent after caller input had started a turn, landing before or over that turn's reply. SendIfAsync now takes the send lock first and evaluates the condition under _sync right before sending; streaming bookkeeping changes only when the piece is actually sent. Every nonempty partial prompt (partialPrompts) now counts as caller activity, so a caller mid-sentence is never asked to repeat or treated as silent. Tests (onebrain host): a reply queued behind a filler goes out whole, a cut-in on a streamed reply stops the rest, and a caller sending partial prompts past the idle timeout is not prompted. The first and last fail on the previous code. Co-Authored-By: Claude Opus 5.5 --- .../Services/ConversationRelaySession.cs | 110 ++++++++++-------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index 3f54fea6c..c41b96359 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -179,6 +179,15 @@ private async Task HandleMessageAsync(string json) var text = GetString(root, "voicePrompt") ?? string.Empty; var last = !root.TryGetProperty("last", out var lastElement) || lastElement.ValueKind != JsonValueKind.False; _partialPrompt.Append(text); + if (!last && text.Length > 0) + { + // With partialPrompts on, the caller is mid-sentence: not silent, and not to be asked to repeat + lock (_sync) + { + _quietSinceUtc = DateTime.UtcNow; + _activity++; + } + } if (last) { var prompt = _partialPrompt.ToString().Trim(); @@ -425,26 +434,15 @@ private async Task WatchResponseAsync(int seqNumber, CancellationToken cancellat private async Task SpeakFillerAsync(int seqNumber, string text) { - await _sendLock.WaitAsync(); - try + // A stream that already began has set _utteranceOpen and may be waiting for the send lock + var sent = await SendIfAsync( + () => !_ended && _seqNum == seqNumber && !_turnReplied && !_utteranceOpen, + new { type = "text", token = text, last = true }); + if (sent) { - // Decided while holding the send lock, so a reply cannot start streaming between the check and the - // filler; a stream that already began has set _utteranceOpen and is waiting for this lock - lock (_sync) - { - if (_ended || _seqNum != seqNumber || _turnReplied || _utteranceOpen) - { - return false; - } - } - await SendLockedAsync(new { type = "text", token = text, last = true }); + MarkSpeaking(EstimateSpeech(text.Length)); } - finally - { - _sendLock.Release(); - } - MarkSpeaking(EstimateSpeech(text.Length)); - return true; + return sent; } private string PickFillerText(List names, List defaultNames, HashSet said) @@ -500,9 +498,12 @@ private async Task WatchIdleAsync(CancellationToken cancellationToken) _quietSinceUtc = DateTime.UtcNow; } - if (askAgain && IsStillQuiet(activity)) + if (askAgain) { - await SpeakPromptAsync($"say-it-again-{Random.Shared.Next(1, 5)}", DefaultSayItAgainText); + // Rechecked once the send lock is held: caller input arriving meanwhile starts a turn, and the + // prompt must not then be said over or after that turn's reply + await SpeakPromptAsync($"say-it-again-{Random.Shared.Next(1, 5)}", DefaultSayItAgainText, + canSpeak: () => !_isProcessing && _activity == activity); } else if (hangUp) { @@ -529,18 +530,19 @@ private async Task FailAsync() /// private async Task StreamTextAsync(string text) { - lock (_sync) + // Decided once the send lock is held: a piece queued behind another send must not go out after the + // caller cut in (_dropStream) or the session ended meanwhile + await SendIfAsync(() => { if (_ended || _dropStream) { - return; + return false; } _streamSegment.Append(text); _utteranceOpen = true; _streamedThisTurn = true; - } - - await SendAsync(new { type = "text", token = text, last = false }); + return true; + }, new { type = "text", token = text, last = false }); } /// @@ -625,40 +627,36 @@ private async Task SpeakIndicationAsync(string indication) /// /// Speaks a prerecorded prompt's configured text so the voice matches; without one, plays the recording. /// - private async Task SpeakPromptAsync(string name, string? fallbackText) + /// Extra condition, checked under once the send lock is held + private async Task SpeakPromptAsync(string name, string? fallbackText, Func? canSpeak = null) { if (_settings.PromptTexts.TryGetValue(name, out var text) && !string.IsNullOrWhiteSpace(text)) { - await SpeakAsync(text); + await SpeakAsync(text, canSpeak); } else if (fallbackText != null) { - await SpeakAsync(fallbackText); + await SpeakAsync(fallbackText, canSpeak); } - else + else if (await SendIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), + new { type = "play", source = $"{_settings.CallbackHost}/twilio/{name}.mp3" })) { - await SendAsync(new { type = "play", source = $"{_settings.CallbackHost}/twilio/{name}.mp3" }); MarkSpeaking(TimeSpan.FromSeconds(3)); } } - private async Task SpeakAsync(string? text) + /// Extra condition, checked under once the send lock is held + private async Task SpeakAsync(string? text, Func? canSpeak = null) { if (string.IsNullOrWhiteSpace(text)) { return; } - lock (_sync) + if (await SendIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), new { type = "text", token = text, last = true })) { - if (_ended) - { - return; - } + MarkSpeaking(EstimateSpeech(text.Length)); } - - await SendAsync(new { type = "text", token = text, last = true }); - MarkSpeaking(EstimateSpeech(text.Length)); } private async Task EndAsync(string action) @@ -683,26 +681,40 @@ private async Task SendEndAsync(string action) await SendAsync(new { type = "end", handoffData = JsonSerializer.Serialize(new { action }) }); } - /// - /// True when the caller has not spoken or pressed a key since was read - /// and nothing else took over. - /// - private bool IsStillQuiet(int activity) + private record PendingInput(string Content, string? Digits); + + private async Task SendAsync(object payload) { - lock (_sync) + await _sendLock.WaitAsync(); + try + { + await SendLockedAsync(payload); + } + finally { - return !_ended && !_isProcessing && _activity == activity; + _sendLock.Release(); } } - private record PendingInput(string Content, string? Digits); - - private async Task SendAsync(object payload) + /// + /// Sends only if still holds once the send lock is held, so a decision taken + /// before waiting for the lock cannot go stale. It runs under and may update session + /// state that must change only when the send goes ahead. + /// + private async Task SendIfAsync(Func canSend, object payload) { await _sendLock.WaitAsync(); try { + lock (_sync) + { + if (!canSend()) + { + return false; + } + } await SendLockedAsync(payload); + return true; } finally { From 31b05ff1716cbb56f636f1a9a40f697a77afebdc Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Mon, 26 Oct 2026 09:25:26 +0800 Subject: [PATCH 7/8] fix(openai): keep personal data out of the output-limit log When a reply hits the output limit the provider logged every tool call's arguments and, on the streaming path, the whole reply. Both can hold the user's personal data (phone numbers, emails, addresses) and the line is a warning that reaches the shared log store, for every agent on this provider. Tool calls are now logged by name and argument length, and the reply by its length and first 200 characters, on all three paths. That still shows which tool the model was writing and how far it got. Co-Authored-By: Claude Opus 5.5 --- .../Chat/ChatCompletionProvider.Chat.cs | 20 ++++++--- .../Routing/OutputLimitLogTests.cs | 42 +++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 tests/BotSharp.Core.UnitTests/Routing/OutputLimitLogTests.cs diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs index 7f91dbbe8..b67b9dc18 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs @@ -56,7 +56,7 @@ private async Task InnerGetChatCompletions(Agent agent, List InnerGetChatCompletionsAsync(Agent agent, } else if (reason == ChatFinishReason.Length) { - _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}, ToolCalls: {DescribeToolCalls(ToLlmToolCalls(value.ToolCalls))}"); + _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{DescribeContent(text)}, ToolCalls: {DescribeToolCalls(ToLlmToolCalls(value.ToolCalls))}"); msg = new RoleDialogModel(AgentRole.Assistant, $"AI response exceeded max output length") { @@ -325,7 +325,7 @@ private async Task InnerGetChatCompletionsStreamingAsync(Agent { // What the model wrote before the limit is replaced by the notice below, so this is the only record of it _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsStreamingAsync)}, Reason: {choice.FinishReason}, Agent: {agent.Name}, " + - $"MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{textStream.GetText()}, ToolCalls: {DescribeToolCalls(ReconstructToolCalls(toolCalls))}"); + $"MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{DescribeContent(textStream.GetText())}, ToolCalls: {DescribeToolCalls(ReconstructToolCalls(toolCalls))}"); } var text = choice.FinishReason == ChatFinishReason.Length ? "Model reached the maximum number of tokens allowed." @@ -767,10 +767,18 @@ private static List ToLlmToolCalls(IEnumerable? toolC .ToList(); /// - /// The tool calls of a reply cut off by the output limit, arguments as far as the model got + /// The tool calls of a reply cut off by the output limit: each name and how long its arguments got. The arguments + /// themselves are left out, since they can hold the user's personal data. /// - private static string DescribeToolCalls(List calls) - => string.Join(", ", calls.Select(x => $"{x.FunctionName}({x.FunctionArgs})")); + internal static string DescribeToolCalls(List calls) + => string.Join(", ", calls.Select(x => $"{x.FunctionName}({x.FunctionArgs?.Length ?? 0} chars)")); + + /// + /// What a reply cut off by the output limit said: its length and only its start, enough to tell what the model was + /// writing without logging the whole reply. + /// + internal static string DescribeContent(string? text) + => string.IsNullOrEmpty(text) ? "(none)" : $"{text.Length} chars: {(text.Length > 200 ? text[..200] + "..." : text)}"; /// /// Rebuilds the tool calls a streaming reply asked for. diff --git a/tests/BotSharp.Core.UnitTests/Routing/OutputLimitLogTests.cs b/tests/BotSharp.Core.UnitTests/Routing/OutputLimitLogTests.cs new file mode 100644 index 000000000..2ce3fc6db --- /dev/null +++ b/tests/BotSharp.Core.UnitTests/Routing/OutputLimitLogTests.cs @@ -0,0 +1,42 @@ +using BotSharp.Abstraction.Functions.Models; +using BotSharp.Plugin.OpenAI.Providers.Chat; +using Xunit; + +namespace BotSharp.Core.UnitTests.Routing; + +/// +/// What the OpenAI provider logs when a reply runs into the output limit. The log has to show what the model was doing +/// (which tools, how far it got) without carrying what it wrote: tool arguments and replies can hold the user's +/// personal data, and the line is a warning that goes to the shared log store. +/// +public class OutputLimitLogTests +{ + [Fact] + public void Tool_calls_are_logged_by_name_and_argument_length_only() + { + var calls = new List + { + new("call_1", "create_request", "{\"contact_phone\":\"675-555-0118\",\"note\":\"gate code 4411\"}"), + new("call_2", "get_result", null) + }; + + var logged = ChatCompletionProvider.DescribeToolCalls(calls); + + Assert.Equal("create_request(56 chars), get_result(0 chars)", logged); + Assert.DoesNotContain("675-555-0118", logged); + } + + [Fact] + public void A_reply_is_logged_by_its_length_and_its_first_200_characters() + { + var reply = new string('a', 200) + "675-555-0118"; + + var logged = ChatCompletionProvider.DescribeContent(reply); + + Assert.Equal($"212 chars: {new string('a', 200)}...", logged); + Assert.DoesNotContain("675-555-0118", logged); + Assert.Equal("5 chars: Hello", ChatCompletionProvider.DescribeContent("Hello")); + Assert.Equal("(none)", ChatCompletionProvider.DescribeContent(null)); + Assert.Equal("(none)", ChatCompletionProvider.DescribeContent(string.Empty)); + } +} From 7efa76059ddc13ed0a9ce05f1e8f2767dbfc11c5 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Mon, 26 Oct 2026 10:27:06 +0800 Subject: [PATCH 8/8] fix(twilio): close ConversationRelay review findings on signatures and send ordering - Refuse the relay handshake when ValidateSignature is on but the auth token is missing, instead of letting every handshake through. - Record how long a frame takes to say before it is sent, so a cut-in that arrives while it plays restarts the silence clock instead of being overwritten. - Close a streamed utterance only once the send lock is held, and recheck the silence when a filler gets the lock, so no filler slips in right after (or inside) a line closed before a tool call. A filler that is no longer due waits for the next silence instead of ending the watchdog for the turn. - Mark the session ended before waiting on the idle watchdog, so a turn finishing during shutdown cannot pick up another caller input. Co-Authored-By: Claude Opus 5.5 --- .../Services/ConversationRelaySession.cs | 141 +++++++++++------- .../Settings/ConversationRelaySetting.cs | 3 +- .../TwilioConversationRelayMiddleware.cs | 7 +- 3 files changed, 95 insertions(+), 56 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs index c41b96359..73692a8f2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs @@ -85,6 +85,7 @@ public async Task RunAsync(CancellationToken cancellationToken) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var idleWatch = WatchIdleAsync(cts.Token); + int dropped; try { await ReceiveLoopAsync(cts.Token); @@ -95,18 +96,18 @@ public async Task RunAsync(CancellationToken cancellationToken) } finally { + // The call is gone: start no new turns, but let a turn already running finish so the conversation is saved. + // Marked before waiting on the idle watchdog, so a turn finishing meanwhile cannot pick up the next input + lock (_sync) + { + _ended = true; + dropped = _pendingInputs.Count; + _pendingInputs.Clear(); + } cts.Cancel(); await SwallowAsync(idleWatch); } - // The call is gone: start no new turns, but let a turn already running finish so the conversation is saved - int dropped; - lock (_sync) - { - _ended = true; - dropped = _pendingInputs.Count; - _pendingInputs.Clear(); - } if (dropped > 0) { _logger.LogInformation($"Dropped {dropped} caller input(s) received after the agent's last turn started; " + @@ -401,48 +402,57 @@ private async Task WatchResponseAsync(int seqNumber, CancellationToken cancellat { var relay = _settings.ConversationRelay; var said = new HashSet(); - for (var step = 0; ; step++) + var step = 0; + while (true) { var interval = TimeSpan.FromSeconds(relay.ResponseWatchdogIntervals[Math.Min(step, relay.ResponseWatchdogIntervals.Count - 1)]); - while (true) + await Task.Delay(WatchdogPollInterval, cancellationToken); + lock (_sync) { - await Task.Delay(WatchdogPollInterval, cancellationToken); - lock (_sync) + if (IsTurnOver(seqNumber)) { - if (_ended || _seqNum != seqNumber || _turnReplied) - { - return; - } - // _quietSinceUtc is the estimated end of what was last said, or the caller's last input - var quietSince = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < _turnStartedUtc ? _turnStartedUtc : _quietSinceUtc; - if (!_utteranceOpen && DateTime.UtcNow - quietSince >= interval) - { - break; - } + return; + } + if (!IsSilentFor(interval)) + { + continue; } } var text = PickFillerText(step == 0 ? relay.ResponseWatchdogPrompts : relay.ResponseWatchdogLongPrompts, step == 0 ? DefaultWatchdogPrompts : DefaultWatchdogLongPrompts, said); - if (!await SpeakFillerAsync(seqNumber, text)) + if (await SpeakFillerAsync(seqNumber, text, interval)) { - return; + said.Add(text); + step++; } - said.Add(text); } } - private async Task SpeakFillerAsync(int seqNumber, string text) + private async Task SpeakFillerAsync(int seqNumber, string text, TimeSpan interval) { - // A stream that already began has set _utteranceOpen and may be waiting for the send lock - var sent = await SendIfAsync( - () => !_ended && _seqNum == seqNumber && !_turnReplied && !_utteranceOpen, - new { type = "text", token = text, last = true }); - if (sent) - { - MarkSpeaking(EstimateSpeech(text.Length)); - } - return sent; + // Decided again once the send lock is held: while waiting for it the reply may have come in, or a streamed + // line may have started or been closed, and then the silence this filler was meant for is over + return await SpeakIfAsync(() => !IsTurnOver(seqNumber) && IsSilentFor(interval), + new { type = "text", token = text, last = true }, EstimateSpeech(text.Length)); + } + + /// + /// The turn the response watchdog was started for has its reply, or the call moved on. Call under . + /// + private bool IsTurnOver(int seqNumber) + { + return _ended || _seqNum != seqNumber || _turnReplied; + } + + /// + /// Nothing has been said, by either side, for . Call under . + /// + private bool IsSilentFor(TimeSpan interval) + { + // _quietSinceUtc is the estimated end of what was last said, or the caller's last input + var quietSince = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < _turnStartedUtc ? _turnStartedUtc : _quietSinceUtc; + return !_utteranceOpen && DateTime.UtcNow - quietSince >= interval; } private string PickFillerText(List names, List defaultNames, HashSet said) @@ -550,21 +560,29 @@ await SendIfAsync(() => /// private async Task CloseUtteranceAsync() { - int length; lock (_sync) { if (!_utteranceOpen) { return; } + } + + // Closed only once the send lock is held, so nothing else (a filler) can go out between the utterance being + // marked closed and its closing frame, and the silence clock already counts the closed line when a filler + // next looks at it + await SendIfAsync(() => + { + if (!_utteranceOpen) + { + return false; + } _utteranceOpen = false; _lastStreamSegment = _streamSegment.ToString(); _streamSegment.Clear(); - length = _lastStreamSegment.Length; - } - - await SendAsync(new { type = "text", token = string.Empty, last = true }); - MarkSpeaking(EstimateSpeech(length)); + MarkSpeaking(EstimateSpeech(_lastStreamSegment.Length)); + return true; + }, new { type = "text", token = string.Empty, last = true }); } /// @@ -638,10 +656,10 @@ private async Task SpeakPromptAsync(string name, string? fallbackText, Func !_ended && (canSpeak?.Invoke() ?? true), - new { type = "play", source = $"{_settings.CallbackHost}/twilio/{name}.mp3" })) + else { - MarkSpeaking(TimeSpan.FromSeconds(3)); + await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), + new { type = "play", source = $"{_settings.CallbackHost}/twilio/{name}.mp3" }, TimeSpan.FromSeconds(3)); } } @@ -653,10 +671,26 @@ private async Task SpeakAsync(string? text, Func? canSpeak = null) return; } - if (await SendIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), new { type = "text", token = text, last = true })) + await SpeakIfAsync(() => !_ended && (canSpeak?.Invoke() ?? true), + new { type = "text", token = text, last = true }, EstimateSpeech(text.Length)); + } + + /// + /// Sends something the caller hears, if still holds once the send lock is held. How long + /// it takes to say is recorded before the frame goes out, so a cut-in arriving once it plays restarts the silence + /// clock instead of being overwritten by it. + /// + private async Task SpeakIfAsync(Func canSpeak, object payload, TimeSpan duration) + { + return await SendIfAsync(() => { - MarkSpeaking(EstimateSpeech(text.Length)); - } + if (!canSpeak()) + { + return false; + } + MarkSpeaking(duration); + return true; + }, payload); } private async Task EndAsync(string action) @@ -743,14 +777,15 @@ private async Task SendLockedAsync(object payload) } } + /// + /// Moves the start of caller silence past something about to be said. Call under , before the + /// frame is sent. + /// private void MarkSpeaking(TimeSpan duration) { - lock (_sync) - { - var now = DateTime.UtcNow; - var start = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < now ? now : _quietSinceUtc; - _quietSinceUtc = start + duration; - } + var now = DateTime.UtcNow; + var start = _quietSinceUtc == DateTime.MaxValue || _quietSinceUtc < now ? now : _quietSinceUtc; + _quietSinceUtc = start + duration; } private static TimeSpan EstimateSpeech(int characters) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs index cb6cbbfe9..d447b3d48 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/ConversationRelaySetting.cs @@ -65,7 +65,8 @@ public class ConversationRelaySetting public int MaxReconnects { get; set; } = 2; /// - /// Verify the X-Twilio-Signature header of the WebSocket handshake + /// Verify the X-Twilio-Signature header of the WebSocket handshake. Needs Twilio:RequestValidation:AuthToken; + /// without it every handshake is refused /// public bool ValidateSignature { get; set; } = true; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs index eb2e863a9..7a9dd8f86 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioConversationRelayMiddleware.cs @@ -69,8 +69,11 @@ private bool IsSignatureValid(HttpContext httpContext, string agentId, string co var authToken = services.GetRequiredService()["Twilio:RequestValidation:AuthToken"]; if (string.IsNullOrEmpty(authToken)) { - _logger.LogWarning("Twilio auth token is not configured; skipping ConversationRelay signature check."); - return true; + // Fail closed: a deployment missing the token must not open the relay to unsigned handshakes + _logger.LogError("Rejected ConversationRelay connection: ConversationRelay.ValidateSignature is on but " + + "Twilio:RequestValidation:AuthToken is not configured. Configure the token, or turn ValidateSignature off " + + "for local testing."); + return false; } var signature = httpContext.Request.Headers["X-Twilio-Signature"].ToString();