Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,9 @@ public static class Gpt4xModelConstants
public const string GPT_4_1_Nano = "gpt-4.1-nano";
public const string GPT_4o_Mini_Realtime_Preview = "gpt-4o-mini-realtime-preview";
public const string GPT_4o_Realtime_Preview = "gpt-4o-realtime-preview";
public const string GPT_4o_Mini_Search_Preview = "gpt-4o-mini-search-preview";
public const string GPT_4o_Mini_Transcribe = "gpt-4o-mini-transcribe";
public const string GPT_4o_Transcribe = "gpt-4o-transcribe";
public const string GPT_4o_Mini_Tts = "gpt-4o-mini-tts";
public const string GPT_4o_Search_Preview = "gpt-4o-search-preview";
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ public class ModelTurnDetection

public class AudioTranscription
{
public string Model { get; set; } = "gpt-4o-mini-transcribe";
public string Model { get; set; } = Gpt4xModelConstants.GPT_4o_Mini_Transcribe;
public string? Language { get; set; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ namespace BotSharp.Abstraction.Realtime.Settings;
public class RealtimeModelSettings
{
public string Provider { get; set; } = "openai";
public string Model { get; set; } = "gpt-4o-mini-realtime-preview";
public string Model { get; set; } = Gpt4xModelConstants.GPT_4o_Mini_Realtime_Preview;
public string[] Modalities { get; set; } = ["text", "audio"];
public bool InterruptResponse { get; set; } = true;
public string InputAudioFormat { get; set; } = "g711_ulaw";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using BotSharp.Abstraction.Realtime.Options;
using BotSharp.Abstraction.Realtime.Settings;
using BotSharp.Abstraction.Routing.Enums;
using BotSharp.Abstraction.Settings;
using BotSharp.Core.Infrastructures;

namespace BotSharp.Core.Realtime.Services;
Expand Down Expand Up @@ -192,14 +193,15 @@ public RealtimeHubConnection SetHubConnection(string conversationId)
{
var provider = agent?.LlmConfig?.Realtime?.Provider;
var model = agent?.LlmConfig?.Realtime?.Model;
var settingService = _services.GetRequiredService<ISettingService>();

if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
return (provider, model);
}

provider = _settings.Provider;
model = _settings.Model;
model = settingService.GetUpgradeModel(_settings.Model);

if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Templating;

namespace BotSharp.Core.Conversations.Services;
Expand Down Expand Up @@ -62,8 +63,10 @@ private async Task<string> Summarize(Agent agent, string prompt)
string? model;

var providerService = _services.GetRequiredService<ILlmProviderService>();
var settingService = _services.GetRequiredService<ISettingService>();
var modelSettings = providerService.GetProviderModels(provider);
var modelSetting = modelSettings.FirstOrDefault(x => x.Name.IsEqualTo("gpt4-turbo") || x.Name.IsEqualTo("gpt-4o"));
var defaultModel = settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o);
var modelSetting = modelSettings.FirstOrDefault(x => x.Name.IsEqualTo(defaultModel));

if (modelSetting != null)
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
using BotSharp.Abstraction.Instructs.Options;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;
using System.IO;

namespace BotSharp.Core.Files.Services;
Expand All @@ -9,11 +11,12 @@ public partial class FileInstructService
{
public async Task<string> ReadImages(string text, IEnumerable<InstructFileModel> images, InstructOptions? options = null)
{
var settingService = _services.GetRequiredService<ISettingService>();
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
var instruction = await RenderAgentTemplate(innerAgentId, options?.TemplateName, options?.Data);
text = RenderText(text, options?.Data);

var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "gpt-4o", multiModal: true);
var completion = CompletionProvider.GetChatCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o), multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = innerAgentId,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.MLTasks.Settings;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;

namespace BotSharp.Core.Infrastructures;

Expand Down Expand Up @@ -140,6 +142,7 @@ public static IAudioTranscription GetAudioTranscriber(
string? provider = null,
string? model = null)
{
var settingService = services.GetRequiredService<ISettingService>();
var completions = services.GetServices<IAudioTranscription>();
var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
if (completer == null)
Expand All @@ -149,7 +152,7 @@ public static IAudioTranscription GetAudioTranscriber(
return default!;
}

completer.SetModelName(model ?? "gpt-4o-mini-transcribe");
completer.SetModelName(model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o_Mini_Transcribe));
return completer;
}

Expand All @@ -158,6 +161,7 @@ public static IAudioSynthesis GetAudioSynthesizer(
string? provider = null,
string? model = null)
{
var settingService = services.GetRequiredService<ISettingService>();
var completions = services.GetServices<IAudioSynthesis>();
var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
if (completer == null)
Expand All @@ -167,7 +171,7 @@ public static IAudioSynthesis GetAudioSynthesizer(
return default!;
}

completer.SetModelName(model ?? "gpt-4o-mini-tts");
completer.SetModelName(model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o_Mini_Tts));
return completer;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using BotSharp.Abstraction.Instructs.Options;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Templating;
using System.Collections;
using System.Reflection;
Expand Down Expand Up @@ -86,9 +88,10 @@ private string BuildInstruction(string instruction, Dictionary<string, object> d

private async Task<RoleDialogModel> GetAiResponse(string text, Agent agent, InstructOptions? options)
{
var settingService = _services.GetRequiredService<ISettingService>();
var dialogs = await BuildDialogs(text, options);
var provider = options?.Provider ?? agent?.LlmConfig?.Provider ?? "openai";
var model = options?.Model ?? agent?.LlmConfig?.Model ?? "gpt-4o";
var model = options?.Model ?? agent?.LlmConfig?.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model);
return await completion.GetChatCompletions(agent, dialogs);
}
Expand Down
5 changes: 4 additions & 1 deletion src/Infrastructure/BotSharp.Core/Shared/JsonRepairService.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.Shared;
using BotSharp.Abstraction.Shared.Options;
using BotSharp.Abstraction.Templating;
Expand Down Expand Up @@ -93,12 +95,13 @@ private async Task<string> RepairByLLMAsync(string malformedJson, JsonRepairOpti
var data = options?.Data ?? new Dictionary<string, object>();
data["input"] = malformedJson;
var prompt = render.Render(template, data);
var settingService = _services.GetRequiredService<ISettingService>();

try
{
var completion = CompletionProvider.GetChatCompletion(_services,
provider: options?.Provider ?? agent?.LlmConfig?.Provider ?? "openai",
model: options?.Model ?? agent?.LlmConfig?.Model ?? "gpt-4o-mini");
model: options?.Model ?? agent?.LlmConfig?.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o_Mini));

var innerAgent = new Agent
{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;

namespace BotSharp.Core.WebSearch.Functions;

Expand Down Expand Up @@ -70,7 +72,8 @@ private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel>
{
var state = _services.GetRequiredService<IConversationStateService>();
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();

var settingService = _services.GetRequiredService<ISettingService>();

var provider = state.GetState("web_search_llm_provider");
var model = state.GetState("web_search_llm_model");

Expand All @@ -80,7 +83,7 @@ private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel>
}

provider = "openai";
model = "gpt-4o-mini-search-preview";
model = settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o_Mini_Search_Preview);

var models = llmProviderService.GetProviderModels(provider);
var foundModel = models.FirstOrDefault(x => x.WebSearch?.IsDefault == true)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;

namespace BotSharp.Plugin.AudioHandler.Functions;

public class ReadAudioFn : IFunctionCallback
Expand Down Expand Up @@ -149,6 +152,7 @@ private bool VerifyAudioFileType(string fileName)
{
var provider = agent?.LlmConfig?.AudioTranscription?.Provider;
var model = agent?.LlmConfig?.AudioTranscription?.Model;
var settingService = _services.GetRequiredService<ISettingService>();

if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
{
Expand All @@ -164,7 +168,7 @@ private bool VerifyAudioFileType(string fileName)
}

provider = "openai";
model = "gpt-4o-mini-transcribe";
model = settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o_Mini_Transcribe);

return (provider, model);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using BotSharp.Abstraction.Messaging.Models.RichContent.Template;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.EmailHandler.Models;
using BotSharp.Plugin.EmailHandler.Providers;
using MailKit;
Expand Down Expand Up @@ -66,8 +68,9 @@ public async Task<bool> Execute(RoleDialogModel message)
};

var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var settingService = _services.GetRequiredService<ISettingService>();
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4o");
var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o));
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model?.Name);
var convService = _services.GetRequiredService<IConversationService>();
var conversationId = convService.ConversationId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using BotSharp.Abstraction.Realtime.Options;
using BotSharp.Abstraction.Realtime.Sessions;
using BotSharp.Abstraction.Realtime.Settings;
using BotSharp.Abstraction.Settings;
using BotSharp.Core.Infrastructures.Streams;
using BotSharp.Core.Session;
using BotSharp.Plugin.GoogleAI.Models.Realtime;
Expand Down Expand Up @@ -87,8 +88,9 @@ public async Task Connect(

var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var settingService = _services.GetRequiredService<ISettingService>();

_model ??= realtimeModelSettings.Model;
_model ??= settingService.GetUpgradeModel(realtimeModelSettings.Model);
var modelSettings = settingsService.GetSetting(Provider, _model);

Reset();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using BotSharp.Abstraction.Models;

namespace BotSharp.Plugin.ImageHandler.Helpers;

internal static class AiResponseHelper
Expand All @@ -7,8 +9,9 @@ internal static async Task<string> GetImageGenerationResponse(IServiceProvider s
var text = $"Please generate a user-friendly response from the following description to " +
$"inform user that you have completed the required image: {description}";

var settingService = services.GetRequiredService<ISettingService>();
var provider = agent?.LlmConfig?.Provider ?? "openai";
var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini";
var model = agent?.LlmConfig?.Model ?? settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o_Mini);
var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model);
var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]);
return response.Content.IfNullOrEmptyAs(GetDefaultResponse(files)) ?? string.Empty;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using BotSharp.Abstraction.Models;

namespace BotSharp.Plugin.OpenAI.Models.Realtime;

public class RealtimeSessionBody
Expand Down Expand Up @@ -82,7 +84,7 @@ public class RealtimeSessionTurnDetection
public class InputAudioTranscription
{
[JsonPropertyName("model")]
public string Model { get; set; } = "gpt-4o-transcribe";
public string Model { get; set; } = Gpt4xModelConstants.GPT_4o_Transcribe;

[JsonPropertyName("language")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Realtime.Options;
using BotSharp.Abstraction.Realtime.Settings;
using BotSharp.Abstraction.Settings;
using OpenAI.Chat;

namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
Expand All @@ -17,7 +19,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
private readonly ILogger<RealTimeCompletionProvider> _logger;
private readonly BotSharpOptions _botsharpOptions;

private string _model = "gpt-4o-mini-realtime-preview";
private string _model = Gpt4xModelConstants.GPT_4o_Mini_Realtime_Preview;
private LlmRealtimeSession _session;
private RealtimeOptions? _realtimeOptions;
private bool _isBlocking = false;
Expand All @@ -40,6 +42,9 @@ public RealTimeCompletionProvider(
_logger = logger;
_services = services;
_botsharpOptions = botsharpOptions;

var settingService = _services.GetRequiredService<ISettingService>();
_model = settingService.GetUpgradeModel(_model);
}

public async Task Connect(
Expand Down Expand Up @@ -67,8 +72,9 @@ public async Task Connect(

var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
var settingService = _services.GetRequiredService<ISettingService>();

_model ??= realtimeSettings.Model;
_model ??= settingService.GetUpgradeModel(realtimeSettings.Model);
var settings = settingsService.GetSetting(Provider, _model);

_session = new LlmRealtimeSession(_services, new ChatSessionOptions
Expand Down Expand Up @@ -321,6 +327,7 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit
{
var convService = _services.GetRequiredService<IConversationService>();
var agentService = _services.GetRequiredService<IAgentService>();
var settingService = _services.GetRequiredService<ISettingService>();

var conv = await convService.GetConversation(conn.ConversationId);
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
Expand Down Expand Up @@ -370,7 +377,7 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit

sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
{
Model = realtimeModelSettings.InputAudioTranscription.Model,
Model = settingService.GetUpgradeModel(realtimeModelSettings.InputAudioTranscription.Model),
Language = realtimeModelSettings.InputAudioTranscription.Language,
Comment on lines 377 to 381
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Action required

1. inputaudiotranscription null dereference 📘 Rule violation ⛯ Reliability

realtimeModelSettings.InputAudioTranscription.Model is dereferenced and passed into
GetUpgradeModel without checking whether InputAudioTranscription (or its Model) is null. This
can throw at runtime when that settings section is missing/partial.
Agent Prompt
## Issue description
The realtime session update dereferences `realtimeModelSettings.InputAudioTranscription` without null checks, then passes the model into `GetUpgradeModel`.

## Issue Context
Configuration sections for realtime input audio transcription can be absent or partially configured; the code should behave predictably (e.g., fall back to a known default model).

## Fix Focus Areas
- src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs[377-381]

ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools

Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ You may obtain a copy of the License at
limitations under the License.
******************************************************************************/

using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Settings;

namespace BotSharp.Plugin.Planner.Sequential;

/// <summary>
Expand Down Expand Up @@ -166,7 +169,8 @@ public async Task<DecomposedStep> GetDecomposedStepAsync(Agent router, string me
var inst = new DecomposedStep();

var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var model = llmProviderService.GetProviderModel("openai", "gpt-4o");
var settingService = _services.GetRequiredService<ISettingService>();
var model = llmProviderService.GetProviderModel("openai", settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o));

// chat completion
var completion = CompletionProvider.GetChatCompletion(_services,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BotSharp.Abstraction.Instructs.Options;
using BotSharp.Abstraction.Models;

namespace BotSharp.Plugin.SqlDriver.Functions;

Expand Down Expand Up @@ -51,7 +52,7 @@ public async Task<bool> Execute(RoleDialogModel message)

if (msgCopy.Data != null && msgCopy.Data is DbException ex)
{

var settingService = _services.GetRequiredService<ISettingService>();
var instructService = _services.GetRequiredService<IInstructService>();
var agentService = _services.GetRequiredService<IAgentService>();
var states = _services.GetRequiredService<IConversationStateService>();
Expand All @@ -63,7 +64,7 @@ public async Task<bool> Execute(RoleDialogModel message)
new InstructOptions
{
Provider = "openai",
Model = "gpt-4o",
Model = settingService.GetUpgradeModel(Gpt4xModelConstants.GPT_4o),
AgentId = BuiltInAgentId.SqlDriver,
TemplateName = "sql_statement_correctness",
Data = new Dictionary<string, object>
Expand Down
Loading
Loading