diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 0bb0257580..792c671ac0 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -41,6 +41,7 @@ DEVSTRAL_MODEL="unsloth/Devstral-Small-2507" LFM2_MODEL="LiquidAI/LFM2-2.6B" LFM25_MODEL="LiquidAI/LFM2.5-8B-A1B" GEMMA4_MODEL="OpenVINO/gemma-4-E4B-it-int4-ov" +MINICPM5_MODEL="openbmb/MiniCPM5-1B" if [ "$(python3 -c 'import sys; print(sys.version_info[1])')" -le "8" ]; then echo "Prepare models with python > 3.8."; exit 1 ; fi @@ -230,6 +231,17 @@ if [ ! -f "$1/$GEMMA4_MODEL/$TOKENIZER_FILE" ]; then exit 1 fi +if [ -f "$1/$MINICPM5_MODEL/$TOKENIZER_FILE" ]; then + echo "Models file $1/$MINICPM5_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." +else + mkdir -p $1/$MINICPM5_MODEL + convert_tokenizer $MINICPM5_MODEL --with_detokenizer -o $1/$MINICPM5_MODEL +fi +if [ ! -f "$1/$MINICPM5_MODEL/$TOKENIZER_FILE" ]; then + echo "[ERROR] Models file $1/$MINICPM5_MODEL/$TOKENIZER_FILE does not exist." + exit 1 +fi + if [ -f "$1/$VLM_MODEL/$TOKENIZER_FILE" ]; then echo "Model file $1/$VLM_MODEL/$TOKENIZER_FILE exists. Skipping downloading models." else @@ -240,4 +252,5 @@ fi if [ ! -f "$1/$VLM_MODEL/$TOKENIZER_FILE" ]; then echo "[ERROR] Model file $1/$VLM_MODEL/$TOKENIZER_FILE does not exist." exit 1 -fi \ No newline at end of file +fi + diff --git a/src/llm/BUILD b/src/llm/BUILD index b3525b17ad..dead0516af 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -275,6 +275,7 @@ ovms_cc_library( "//src/port:rapidjson_writer", "//src/port:rapidjson_document", ":io_processing_utils", + ":apis_tool_schema_wrapper", "//third_party:genai", ], ) @@ -360,6 +361,29 @@ ovms_cc_library( visibility = ["//visibility:public"], ) +ovms_cc_library( + name = "io_processing_minicpm5_tool_parser", + hdrs = [ + "io_processing/minicpm5/minicpm5_tool_parser.hpp", + "io_processing/minicpm5/minicpm5_reasoning_parser.hpp", + ], + srcs = [ + "io_processing/minicpm5/minicpm5_tool_parser.cpp", + "io_processing/minicpm5/minicpm5_reasoning_parser.cpp", + ], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src:libovmslogging", + "//src:libovmsstatus", + "//src/utils:rapidjson_utils", + ":io_processing_utils", + ":io_processing_base_output_parser", + ":apis_tool_schema_wrapper", + "//third_party:genai", + ], + visibility = ["//visibility:public"], +) + ovms_cc_library( name = "io_processing_qwen3_reasoning_parser", hdrs = ["io_processing/qwen3/reasoning_parser.hpp"], @@ -429,6 +453,7 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w ":io_processing_lfm2_tool_parser", ":io_processing_lfm25_tool_parser", ":io_processing_gemma4_tool_parser", + ":io_processing_minicpm5_tool_parser", ":io_processing_qwen3_reasoning_parser", ":io_processing_lfm25_reasoning_parser", ":io_processing_utils", diff --git a/src/llm/io_processing/base_output_parser.cpp b/src/llm/io_processing/base_output_parser.cpp index 20fa3854f4..2091488779 100644 --- a/src/llm/io_processing/base_output_parser.cpp +++ b/src/llm/io_processing/base_output_parser.cpp @@ -28,6 +28,50 @@ namespace ovms { +ParametersTypeMap_t parseToolSchema(const rapidjson::Value& schema) { + // Map each declared parameter name to its ParameterType from the tool's JSON schema. + ParametersTypeMap_t result; + if (!schema.IsObject()) { + return result; + } + if (!schema.HasMember("properties") || !schema["properties"].IsObject()) { + return result; + } + const rapidjson::Value& properties = schema["properties"]; + for (auto it = properties.MemberBegin(); it != properties.MemberEnd(); ++it) { + if (!it->value.IsObject()) { + continue; + } + if (!it->value.HasMember("type") || !it->value["type"].IsString()) { + continue; + } + std::string paramName = it->name.GetString(); + std::string typeStr = it->value["type"].GetString(); + ParameterType type = ParameterType::UNKNOWN; + if (typeStr == "string") { + type = ParameterType::STRING; + } else if (typeStr == "number" || typeStr == "integer") { + type = ParameterType::NUMBER; + } else if (typeStr == "boolean") { + type = ParameterType::BOOLEAN; + } else if (typeStr == "array") { + type = ParameterType::ARRAY; + } else if (typeStr == "object") { + type = ParameterType::OBJECT; + } + result.emplace(paramName, type); + } + return result; +} + +ToolsParameterTypeMap_t createToolsParametersTypesMap(const ToolsSchemas_t& toolsSchemas) { + ToolsParameterTypeMap_t toolsParametersTypes; + for (const auto& [toolName, toolSchemaWrapper] : toolsSchemas) { + toolsParametersTypes.emplace(toolName, parseToolSchema(*toolSchemaWrapper.rapidjsonRepr)); + } + return toolsParametersTypes; +} + rapidjson::Document BaseOutputParser::wrapFirstDelta(const std::string& functionName, int toolCallIndex) { rapidjson::Document wrappedDelta; wrappedDelta.SetObject(); diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index 59018ba28b..0b83ea3839 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -29,6 +29,8 @@ #include "src/port/rapidjson_stringbuffer.hpp" #include "src/port/rapidjson_writer.hpp" +#include "src/llm/apis/tool_schema_wrapper.hpp" + namespace ovms { struct ToolCall { std::string id; @@ -58,6 +60,12 @@ enum class ParameterType { using ParametersTypeMap_t = std::unordered_map; // param name -> param type using ToolsParameterTypeMap_t = std::unordered_map; // tool name -> (param name -> param type) +// Tool-schema helpers shared between tag/attribute-style parsers (e.g. qwen3coder, minicpm5). +// Builds a parameter name -> ParameterType map from a single tool's JSON schema. +ParametersTypeMap_t parseToolSchema(const rapidjson::Value& schema); +// Builds a tool name -> (parameter name -> ParameterType) map from all tools' schemas. +ToolsParameterTypeMap_t createToolsParametersTypesMap(const ToolsSchemas_t& toolsSchemas); + class BaseOutputParser { protected: ov::genai::Tokenizer tokenizer; diff --git a/src/llm/io_processing/chat_template/analyzer.cpp b/src/llm/io_processing/chat_template/analyzer.cpp index 29d4ab1801..a2e8a0f82c 100644 --- a/src/llm/io_processing/chat_template/analyzer.cpp +++ b/src/llm/io_processing/chat_template/analyzer.cpp @@ -56,6 +56,15 @@ ChatTemplateAnalysisResult ChatTemplateAnalyzer::analyze(const std::string& temp return result; } + // MiniCPM5 detection — uses XML style (distinct from qwen3coder's ) + if (contains(templateSource, "") || contains(templateSource, "<|tool_call_start|>") || contains(templateSource, "keep_past_thinking")) { result.detectedToolParser = "lfm2"; diff --git a/src/llm/io_processing/chat_template/probe.cpp b/src/llm/io_processing/chat_template/probe.cpp index fe4a73dab2..f2b068b46d 100644 --- a/src/llm/io_processing/chat_template/probe.cpp +++ b/src/llm/io_processing/chat_template/probe.cpp @@ -96,6 +96,7 @@ static bool analyzeProbeToolArgumentResults(bool strOk, const std::string& strOu return output.find("\"" + PROBE_NEEDLE + "\": \"") != std::string::npos || output.find("\"" + PROBE_NEEDLE + "\":\"") != std::string::npos || output.find("") != std::string::npos || + output.find("") != std::string::npos || output.find(PROBE_NEEDLE + ":<|") != std::string::npos || output.find(PROBE_NEEDLE + "=") != std::string::npos; }; diff --git a/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.cpp b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.cpp new file mode 100644 index 0000000000..11c4218e1e --- /dev/null +++ b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.cpp @@ -0,0 +1,81 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/logging.hpp" +#include "minicpm5_reasoning_parser.hpp" +#include "src/llm/io_processing/utils.hpp" + +namespace ovms { +void Minicpm5ReasoningParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + auto startReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningStartTokenId); + auto endReasoningIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId); + + if ((startReasoningIt == generatedTokens.end() && endReasoningIt == generatedTokens.end())) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ReasoningParser: Reasoning start or end token not found in the generated tokens. Start token found: {}, End token found: {}, Start position: {}, End position: {}", + startReasoningIt != generatedTokens.end(), endReasoningIt != generatedTokens.end(), std::distance(generatedTokens.begin(), startReasoningIt), std::distance(generatedTokens.begin(), endReasoningIt)); + return; + } + + auto startPos = 0; + if (startReasoningIt != generatedTokens.end()) { + startPos = std::distance(generatedTokens.begin(), startReasoningIt) + 1; + } else { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ReasoningParser: Reasoning start token not found in the generated tokens. Start position: {}", startPos); + } + auto endPos = std::distance(generatedTokens.begin(), endReasoningIt); + + std::string reasoningContent = tokenizer.decode(std::vector(startPos + generatedTokens.begin(), endPos + generatedTokens.begin()), ov::genai::skip_special_tokens(true)); + + parsedOutput.reasoning = reasoningContent; + + if (endReasoningIt != generatedTokens.end()) { + endPos += 1; + } + + std::string contentWithoutReasoning = tokenizer.decode(std::vector(endPos + generatedTokens.begin(), generatedTokens.end()), ov::genai::skip_special_tokens(true)); + parsedOutput.content = contentWithoutReasoning; +} + +std::optional Minicpm5ReasoningParser::parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) { + if (tokens.empty()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Received empty tokens for Minicpm5ReasoningParser"); + return std::nullopt; + } + + if (std::find(tokens.begin(), tokens.end(), reasoningStartTokenId) != tokens.end() || + std::find(tokens.begin(), tokens.end(), reasoningEndTokenId) != tokens.end()) { + return std::nullopt; + } else { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + writer.String("delta"); + writer.StartObject(); + writer.String("reasoning_content"); + writer.String(chunk.c_str()); + writer.EndObject(); + writer.EndObject(); + rapidjson::Document doc; + doc.Parse(buffer.GetString()); + return doc; + } +} +} // namespace ovms diff --git a/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp new file mode 100644 index 0000000000..fe194638f8 --- /dev/null +++ b/src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp @@ -0,0 +1,53 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include "src/llm/io_processing/base_output_parser.hpp" +#include +#include + +namespace ovms { +class Minicpm5ReasoningParser : public BaseOutputParser { +public: + static inline const std::string reasoningStartTag = ""; + static inline const std::string reasoningEndTag = ""; + + static constexpr int64_t reasoningStartTokenId = 8; + static constexpr int64_t reasoningEndTokenId = 9; + +public: + Minicpm5ReasoningParser() = delete; + explicit Minicpm5ReasoningParser(ov::genai::Tokenizer& tokenizer) : + BaseOutputParser(tokenizer) {} + + void parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) override; + std::optional parseChunk(const std::string& chunk, const std::vector& tokens, ov::genai::GenerationFinishReason finishReason) override; + const std::vector& getParsingStartTags() const override { + static const std::vector parsingStartTags{this->reasoningStartTag}; + return parsingStartTags; + } + const std::vector& getSpecialParsingStartTags() const override { + static const std::vector specialParsingStartTags{}; + return specialParsingStartTags; + } + const std::string& getParsingEndTag() const override { + return reasoningEndTag; + } + + bool requiresStreamingWithSpecialTokens() const override { + return true; + } +}; +} // namespace ovms diff --git a/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp new file mode 100644 index 0000000000..08e19c27a3 --- /dev/null +++ b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.cpp @@ -0,0 +1,438 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include +#include +#include + +#include "rapidjson/error/en.h" + +#include "src/llm/io_processing/utils.hpp" +#include "src/logging.hpp" +#include "src/utils/rapidjson_utils.hpp" +#include "minicpm5_tool_parser.hpp" + +namespace ovms { + +// ---- Tag string constants ---- +const std::string Minicpm5ToolParser::FUNCTION_START_TAG = ""; +const std::string Minicpm5ToolParser::PARAM_START_TAG = "' + size_t end = content.find_first_of(" \t\n\r>/", nameAttrValueStart); + if (end == std::string::npos || end > tagEnd) + end = tagEnd; + return content.substr(nameAttrValueStart, end - nameAttrValueStart); + } + size_t closeQuote = content.find(quote, nameAttrValueStart + 1); + if (closeQuote == std::string::npos || closeQuote > tagEnd) { + return {}; + } + return content.substr(nameAttrValueStart + 1, closeQuote - nameAttrValueStart - 1); +} + +void Minicpm5ToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameterValueAsString) { + if (this->removeNewlineAroundParameters) + trimNewline(parameterValueAsString); + + auto paramIt = this->toolsParametersTypeMap.find(this->currentFunction.name); + auto& currentFunctionArgsDoc = this->currentFunction.argumentsAsDocument; + auto& allocator = currentFunctionArgsDoc.GetAllocator(); + auto& key = this->currentParameterName; + rapidjson::Value keyVal(key.c_str(), allocator); + rapidjson::Value valueCopy; + + rapidjson::Document temp; + // Boolean normalisation (shared helper, same as qwen3coder) + if (paramIt != this->toolsParametersTypeMap.end()) { + auto paramJt = paramIt->second.find(currentParameterName); + if (paramJt != paramIt->second.end() && paramJt->second == ParameterType::BOOLEAN) { + normalizeBooleanString(parameterValueAsString); + } + } + + temp.Parse(parameterValueAsString.c_str()); + rapidjson::Document retryDoc; + bool parsingSucceeded = !temp.HasParseError(); + + if (!parsingSucceeded) { + if (!parameterValueAsString.empty() && + (parameterValueAsString.front() == '{' || parameterValueAsString.front() == '[')) { + std::string converted = replaceSingleWithDoubleQuotes(parameterValueAsString); + retryDoc.Parse(converted.c_str()); + if (!retryDoc.HasParseError()) { + SPDLOG_TRACE("Minicpm5: successfully parsed after single-to-double quote conversion: {}", converted); + parameterValueAsString = std::move(converted); + valueCopy.CopyFrom(retryDoc, allocator); + parsingSucceeded = true; + } + } + if (!parsingSucceeded) { + rapidjson::ParseErrorCode errorCode = temp.GetParseError(); + size_t errorOffset = temp.GetErrorOffset(); + SPDLOG_TRACE("Minicpm5: RapidJSON cannot parse param: {} value: {}; error offset: {}; code: {}; falling back to string", + this->currentParameterName, parameterValueAsString, errorOffset, rapidjson::GetParseError_En(errorCode)); + valueCopy.SetString(parameterValueAsString.c_str(), static_cast(parameterValueAsString.size()), allocator); + } + } else { + valueCopy.CopyFrom(temp, allocator); + if (paramIt != this->toolsParametersTypeMap.end()) { + auto paramJt = paramIt->second.find(currentParameterName); + if (paramJt != paramIt->second.end() && paramJt->second == ParameterType::STRING) { + enforceStringValue(valueCopy, allocator); + } + } + } + if (!currentFunctionArgsDoc.HasMember(keyVal)) { + currentFunctionArgsDoc.AddMember(keyVal, valueCopy, allocator); + } else { + SPDLOG_DEBUG("Minicpm5: parameter {} already exists in document", key); + } +} + +Status Minicpm5ToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { + if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { + SPDLOG_DEBUG("Minicpm5: mismatched tool tags, begin: {}, end: {}", + toolCallPositions.begin.size(), toolCallPositions.end.size()); + return Status(StatusCode::INTERNAL_ERROR, "Mismatched tool tags"); + } + while (!toolCallPositions.begin.empty() && !toolCallPositions.end.empty()) { + auto posBegin = toolCallPositions.begin.top(); + auto posEnd = toolCallPositions.end.top(); + SPDLOG_TRACE("Minicpm5: removing tool call from outContent begin:{}, end:{}", posBegin, posEnd); + outContent.erase(posBegin, posEnd - posBegin); + toolCallPositions.begin.pop(); + toolCallPositions.end.pop(); + } + + const std::vector tokensToErase = { + Minicpm5ToolParser::SOS_TOKEN_STR, + Minicpm5ToolParser::EOS_TOKEN_STR}; + + for (const auto& token : tokensToErase) { + size_t pos = 0; + while ((pos = outContent.find(token, pos)) != std::string::npos) { + outContent.erase(pos, token.length()); + } + } + + return StatusCode::OK; +} + +void Minicpm5ToolParserImpl::handleInsideContentState() { + // Look for the next streamContent.find(Minicpm5ToolParser::FUNCTION_START_TAG, this->lastProcessedPosition); + if (posFunc == std::string::npos) { + SPDLOG_TRACE("Minicpm5: no found"); + return; + } + this->toolCallPositions.begin.push(posFunc); + // Skip past "lastProcessedPosition = posFunc + Minicpm5ToolParser::FUNCTION_START_TAG.size(); + this->currentState = State::InsideFunctionName; +} + +void Minicpm5ToolParserImpl::handleInsideFunctionNameState() { + auto pos = this->streamContent.find(Minicpm5ToolParser::XML_TAG_END, this->lastProcessedPosition); + if (pos == std::string::npos) { + SPDLOG_TRACE("Minicpm5: waiting for '>' of tag"); + return; + } + this->currentFunction.name = this->streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + Minicpm5ToolParser::XML_TAG_END.length(); + this->currentState = State::InsideFunction; +} + +void Minicpm5ToolParserImpl::handleInsideFunctionState(ToolCalls_t& toolCalls) { + // Expect either + auto funcEnd = this->streamContent.find(Minicpm5ToolParser::FUNCTION_END_TAG, this->lastProcessedPosition); + auto paramStart = this->streamContent.find(Minicpm5ToolParser::PARAM_START_TAG, this->lastProcessedPosition); + if (funcEnd == std::string::npos && paramStart == std::string::npos) { + // Waiting for more data + } else if (paramStart != std::string::npos && (funcEnd == std::string::npos || paramStart < funcEnd)) { + // Next lastProcessedPosition = paramStart + Minicpm5ToolParser::PARAM_START_TAG.size(); + this->currentState = State::InsideParamName; + } else { + // + this->currentState = State::AfterFunction; + } +} + +void Minicpm5ToolParserImpl::handleInsideAfterFunctionState(ToolCalls_t& toolCalls) { + auto funcEnd = this->streamContent.find(Minicpm5ToolParser::FUNCTION_END_TAG, this->lastProcessedPosition); + this->lastProcessedPosition = funcEnd + Minicpm5ToolParser::FUNCTION_END_TAG.size(); + this->currentState = State::Content; + std::string argumentsAsString; + { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + this->currentFunction.argumentsAsDocument.Accept(writer); + argumentsAsString = buffer.GetString(); + } + ToolCall toolCall{generateRandomId(), this->currentFunction.name, argumentsAsString}; + SPDLOG_TRACE("Minicpm5: adding tool call: id={}, name={}, params={}", toolCall.id, toolCall.name, toolCall.arguments); + toolCalls.emplace_back(std::move(toolCall)); + this->currentFunction.clear(); + this->toolCallPositions.end.push(this->lastProcessedPosition); +} + +void Minicpm5ToolParserImpl::handleInsideParamNameState() { + auto pos = this->streamContent.find(Minicpm5ToolParser::XML_TAG_END, this->lastProcessedPosition); + if (pos == std::string::npos) { + SPDLOG_TRACE("Minicpm5: waiting for '>' of tag"); + return; + } + this->currentParameterName = streamContent.substr(this->lastProcessedPosition, pos - this->lastProcessedPosition); + this->lastProcessedPosition = pos + Minicpm5ToolParser::XML_TAG_END.length(); + this->currentState = State::InsideParam; +} + +void Minicpm5ToolParserImpl::handleInsideParamState() { + // Read until + auto endPos = this->streamContent.find(Minicpm5ToolParser::PARAM_END_TAG, this->lastProcessedPosition); + if (endPos == std::string::npos) { + SPDLOG_TRACE("Minicpm5: waiting for "); + return; + } + std::string paramValue = this->streamContent.substr(this->lastProcessedPosition, endPos - this->lastProcessedPosition); + SPDLOG_TRACE("Minicpm5: adding parameter {} with value {}", this->currentParameterName, paramValue); + addParameterToCurrentFunctionDoc(paramValue); + this->lastProcessedPosition = endPos + Minicpm5ToolParser::PARAM_END_TAG.size(); + this->currentState = State::InsideFunction; +} + +bool Minicpm5ToolParserImpl::parseUntilStateChange(ToolCalls_t& toolCalls) { + SPDLOG_TRACE("Minicpm5: state: {}", this->currentState); + auto previousState = this->currentState; + + switch (this->currentState) { + case State::Content: + handleInsideContentState(); + break; + case State::InsideFunctionName: + handleInsideFunctionNameState(); + break; + case State::InsideFunction: + handleInsideFunctionState(toolCalls); + break; + case State::InsideParamName: + handleInsideParamNameState(); + break; + case State::InsideParam: + handleInsideParamState(); + break; + case State::AfterFunction: + handleInsideAfterFunctionState(toolCalls); + break; + } + + return previousState != this->currentState; +} + +std::optional Minicpm5ToolParserImpl::parseChunk(const std::string& chunk) { + if (chunk.empty()) + return std::nullopt; + ToolCalls_t toolCalls; + this->streamContent += chunk; + while (parseUntilStateChange(toolCalls)) { + } + if (!toolCalls.empty()) { + return std::move(toolCalls); + } + return std::nullopt; +} + +std::optional Minicpm5ToolParserImpl::getCurrentFunctionName() const { + if (this->currentFunction.name.empty()) + return std::nullopt; + return this->currentFunction.name; +} + +// ---- Minicpm5ToolParser ---- + +void Minicpm5ToolParser::lazyFillInitToolParametersTypesMap() { + if (this->filledParametersTypesMap) + return; + SPDLOG_DEBUG("Minicpm5ToolParser: filling tools parameters types map"); + this->toolsParametersTypes = createToolsParametersTypesMap(this->toolSchemas); + this->filledParametersTypesMap = true; + SPDLOG_DEBUG("Minicpm5ToolParser: created with {} tools", this->toolsParametersTypes.size()); +} + +Minicpm5ToolParser::Minicpm5ToolParser(ov::genai::Tokenizer& tokenizer, const ToolsSchemas_t& toolSchemas) : + BaseOutputParser(tokenizer), + toolSchemas(toolSchemas), + streamParser(this->toolsParametersTypes) {} + +const std::vector Minicpm5ToolParser::removeReasoningTokens(const std::vector& generatedTokens) { + std::vector tokensWithoutReasoning; + tokensWithoutReasoning.reserve(generatedTokens.size()); + auto reasoningStartIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningStartTokenId); + auto reasoningEndIt = std::find(generatedTokens.begin(), generatedTokens.end(), reasoningEndTokenId); + if (reasoningStartIt == generatedTokens.end() && reasoningEndIt == generatedTokens.end()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ToolParser: Reasoning start or end token not found in the generated tokens. Start token found: {}, End token found: {}, Start position: {}, End position: {}", + reasoningStartIt != generatedTokens.end(), reasoningEndIt != generatedTokens.end(), std::distance(generatedTokens.begin(), reasoningStartIt), std::distance(generatedTokens.begin(), reasoningEndIt)); + tokensWithoutReasoning.insert(tokensWithoutReasoning.end(), generatedTokens.begin(), generatedTokens.end()); + } else { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ToolParser: Reasoning tokens found. Start position: {}, End position: {}", + std::distance(generatedTokens.begin(), reasoningStartIt), std::distance(generatedTokens.begin(), reasoningEndIt)); + if (reasoningStartIt == generatedTokens.end()) { + SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Minicpm5ToolParser: Reasoning start wasn't found, but reasoning end was found. Start position: {}, End position: {}", + std::distance(generatedTokens.begin(), reasoningStartIt), std::distance(generatedTokens.begin(), reasoningEndIt)); + reasoningStartIt = generatedTokens.begin(); + } + tokensWithoutReasoning.insert(tokensWithoutReasoning.end(), generatedTokens.begin(), reasoningStartIt); + tokensWithoutReasoning.insert(tokensWithoutReasoning.end(), reasoningEndIt + 1, generatedTokens.end()); + } + return tokensWithoutReasoning; +} + +void Minicpm5ToolParser::parse(ParsedOutput& parsedOutput, const std::vector& generatedTokens) { + auto tokensWithoutReasoning = this->removeReasoningTokens(generatedTokens); + std::string contentWithSpecialTokens = this->tokenizer.decode(tokensWithoutReasoning, ov::genai::skip_special_tokens(false)); + this->lazyFillInitToolParametersTypesMap(); + auto toolCallsOpt = this->streamParser.parseChunk(contentWithSpecialTokens); + if (toolCallsOpt.has_value()) { + parsedOutput.toolCalls = std::move(toolCallsOpt.value()); + SPDLOG_DEBUG("Minicpm5ToolParser: parse done, removing tool calls from content"); + auto status = this->streamParser.removeToolCallsFromContentIfNeeded(contentWithSpecialTokens); + if (!status.ok()) { + SPDLOG_DEBUG("Minicpm5ToolParser: failed to remove tool calls from content: {}", status.string()); + } + parsedOutput.content = std::move(contentWithSpecialTokens); + return; + } + SPDLOG_DEBUG("Minicpm5ToolParser: parse done, no tool calls found"); +} + +std::optional Minicpm5ToolParser::sendFullDelta(const ToolCalls_t& toolCalls) { + if (toolCalls.size() != 1) { + SPDLOG_ERROR("Minicpm5ToolParser: for streaming expected one tool call, got: {}", toolCalls.size()); + throw std::runtime_error("Minicpm5ToolParser: for streaming expected one tool call"); + } + auto& toolCall = toolCalls[0]; + // If the first delta was not sent yet (complete tool call in a single chunk), + // return a combined delta with id, type, name AND arguments. + if (this->returnedFirstDeltas.find(this->toolCallIndex) == this->returnedFirstDeltas.end() || + this->toolCallIndex == -1) { + int toolCallId = ++this->toolCallIndex; + this->returnedFirstDeltas.insert(toolCallId); + this->returnedCompleteDeltas.insert(toolCallId); + return wrapCombinedDelta(toolCall); + } + this->returnedCompleteDeltas.insert(this->toolCallIndex); + rapidjson::Document argumentsWrapper; + argumentsWrapper.SetObject(); + rapidjson::Document::AllocatorType& allocator = argumentsWrapper.GetAllocator(); + rapidjson::Value toolCallsString(rapidjson::kStringType); + toolCallsString.SetString(toolCall.arguments.c_str(), allocator); + SPDLOG_TRACE("Minicpm5ToolParser: tool call arguments string: {}", toolCall.arguments); + argumentsWrapper.AddMember("arguments", toolCallsString, allocator); + auto currentDelta = wrapDelta(argumentsWrapper, this->toolCallIndex); + SPDLOG_DEBUG("Minicpm5ToolParser: full delta: {}", documentToString(currentDelta)); + return currentDelta; +} + +rapidjson::Document Minicpm5ToolParser::wrapCombinedDelta(const ToolCall& toolCall) { + rapidjson::Document wrappedDelta; + wrappedDelta.SetObject(); + rapidjson::Document::AllocatorType& allocator = wrappedDelta.GetAllocator(); + + rapidjson::Value toolCalls(rapidjson::kArrayType); + rapidjson::Value toolCallObj(rapidjson::kObjectType); + rapidjson::Value idValue(generateRandomId().c_str(), allocator); + toolCallObj.AddMember("id", idValue, allocator); + toolCallObj.AddMember("type", "function", allocator); + toolCallObj.AddMember("index", this->toolCallIndex, allocator); + + rapidjson::Value functionObj(rapidjson::kObjectType); + rapidjson::Value nameValue(toolCall.name.c_str(), allocator); + functionObj.AddMember("name", nameValue, allocator); + + rapidjson::Value argumentsValue(rapidjson::kStringType); + argumentsValue.SetString(toolCall.arguments.c_str(), allocator); + functionObj.AddMember("arguments", argumentsValue, allocator); + toolCallObj.AddMember("function", functionObj, allocator); + + toolCalls.PushBack(toolCallObj, allocator); + rapidjson::Value deltaWrapper(rapidjson::kObjectType); + deltaWrapper.AddMember("tool_calls", toolCalls, allocator); + wrappedDelta.AddMember("delta", deltaWrapper, allocator); + SPDLOG_DEBUG("Minicpm5ToolParser: combined delta: {}", documentToString(wrappedDelta)); + return wrappedDelta; +} + +std::optional Minicpm5ToolParser::sendFirstDeltaIfNeeded(const std::string& toolCallName) { + if (this->returnedFirstDeltas.size() == (this->returnedCompleteDeltas.size() + 1)) { + SPDLOG_TRACE("Minicpm5ToolParser: skipping first delta, already sent for current function"); + return std::nullopt; + } + int toolCallId = ++this->toolCallIndex; + rapidjson::Document doc = wrapFirstDelta(toolCallName, toolCallId); + this->currentJson.CopyFrom(doc, this->currentJson.GetAllocator()); + this->returnedFirstDeltas.insert(toolCallId); + SPDLOG_DEBUG("Minicpm5ToolParser: first delta: {}", documentToString(doc)); + return doc; +} + +std::optional Minicpm5ToolParser::parseChunk( + const std::string& newChunk, + const std::vector& /*tokens*/, + ov::genai::GenerationFinishReason /*finishReason*/) { + SPDLOG_DEBUG("Minicpm5ToolParser: chunk: '{}'", newChunk); + this->lazyFillInitToolParametersTypesMap(); + if (newChunk.empty()) + return std::nullopt; + auto toolCallsOpt = this->streamParser.parseChunk(newChunk); + if (toolCallsOpt.has_value()) { + return this->sendFullDelta(toolCallsOpt.value()); + } + auto functionNameOpt = this->streamParser.getCurrentFunctionName(); + if (functionNameOpt.has_value()) { + return this->sendFirstDeltaIfNeeded(functionNameOpt.value()); + } + return std::nullopt; +} + +} // namespace ovms diff --git a/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp new file mode 100644 index 0000000000..4e1cf972c0 --- /dev/null +++ b/src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp @@ -0,0 +1,204 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "src/port/rapidjson_document.hpp" + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/apis/tool_schema_wrapper.hpp" +#include "src/logging.hpp" +#include "src/status.hpp" + +namespace ovms { + +// MiniCPM5 tool call format (attribute-style XML): +// Beijingcelsius +// +// Multiple ... blocks may appear concatenated. +// Reasoning (...) is stripped earlier in the chain by the reasoning parser, +// so this tool parser only deals with blocks. + +struct Minicpm5Functool { + std::string name; + void clear() { + name.clear(); + argumentsAsDocument.SetObject(); + } + rapidjson::Document argumentsAsDocument; + Minicpm5Functool() { + argumentsAsDocument.SetObject(); + } +}; + +struct Minicpm5ToolParserImpl { + enum class State { + Content, // (C) looking for + InsideFunction, // (IF) inside function body — looking for + InsideParamName, // (IPN) inside + InsideParam, // (IP) inside param value — reading until + AfterFunction, // (AF) after — ready to emit tool call + }; + // STATE DEMARKATION + /* + Content + <- InsideFunctionName reads up to > + InsideFunction + ( <- InsideParamName reads up to > + InsideParamInsideFunction)* + Content + */ + + explicit Minicpm5ToolParserImpl(const ToolsParameterTypeMap_t& toolsParametersTypeMap); + + /* + * Feed a chunk; returns any completed tool calls found so far. + */ + std::optional parseChunk(const std::string& chunk); + + std::optional getCurrentFunctionName() const; + + Status removeToolCallsFromContentIfNeeded(std::string& outContent); + + State getCurrentState() const { return this->currentState; } + size_t getLastProcessedPosition() const { return this->lastProcessedPosition; } + +private: + const ToolsParameterTypeMap_t& toolsParametersTypeMap; + const bool removeNewlineAroundParameters = true; + State currentState = State::Content; + Minicpm5Functool currentFunction; + std::string currentParameterName; + std::string streamContent; + size_t lastProcessedPosition{0}; + + struct ToolCallPositions { + std::stack begin; + std::stack end; + }; + ToolCallPositions toolCallPositions; + + void addParameterToCurrentFunctionDoc(std::string& parameterValueAsString); + + bool parseUntilStateChange(ToolCalls_t& toolCalls); + + void handleInsideContentState(); + void handleInsideFunctionNameState(); + void handleInsideFunctionState(ToolCalls_t& toolCalls); + void handleInsideParamNameState(); + void handleInsideParamState(); + void handleInsideAfterFunctionState(ToolCalls_t& toolCalls); + + static std::string extractNameAttribute(const std::string& content, size_t nameAttrValueStart, size_t tagEnd); +}; + +class Minicpm5ToolParser : public BaseOutputParser { +public: + // Tag literals used by the state machine + static const std::string FUNCTION_START_TAG; // & getSpecialTagsToErase() const override { + static const std::vector tagsToErase = {SOS_TOKEN_STR, EOS_TOKEN_STR}; + return tagsToErase; + } + +private: + const std::vector removeReasoningTokens(const std::vector& generatedTokens); + std::optional sendFirstDeltaIfNeeded(const std::string& currentFunctionName); + std::optional sendFullDelta(const ToolCalls_t& toolCalls); + rapidjson::Document wrapCombinedDelta(const ToolCall& toolCall); + void lazyFillInitToolParametersTypesMap(); +}; + +} // namespace ovms + +template <> +struct fmt::formatter : fmt::formatter { + auto format(const ovms::Minicpm5ToolParserImpl::State& state, fmt::format_context& ctx) const { + std::unordered_map stateMap = { + {ovms::Minicpm5ToolParserImpl::State::Content, "Content"}, + {ovms::Minicpm5ToolParserImpl::State::InsideFunctionName, "InsideFunctionName"}, + {ovms::Minicpm5ToolParserImpl::State::InsideFunction, "InsideFunction"}, + {ovms::Minicpm5ToolParserImpl::State::InsideParamName, "InsideParamName"}, + {ovms::Minicpm5ToolParserImpl::State::InsideParam, "InsideParam"}, + {ovms::Minicpm5ToolParserImpl::State::AfterFunction, "AfterFunction"}, + }; + auto it = stateMap.find(state); + if (it != stateMap.end()) { + return fmt::formatter::format(it->second, ctx); + } else { + return fmt::formatter::format("Unknown", ctx); + } + } +}; diff --git a/src/llm/io_processing/output_parser.cpp b/src/llm/io_processing/output_parser.cpp index 679ed8425d..6bcf48bae5 100644 --- a/src/llm/io_processing/output_parser.cpp +++ b/src/llm/io_processing/output_parser.cpp @@ -35,6 +35,8 @@ #include "lfm2/lfm25_tool_parser.hpp" #include "lfm2/lfm25_reasoning_parser.hpp" #include "gemma4/gemma4_tool_parser.hpp" +#include "minicpm5/minicpm5_tool_parser.hpp" +#include "minicpm5/minicpm5_reasoning_parser.hpp" namespace ovms { OutputParser::TagLookupStatus OutputParser::StreamOutputCache::lookupTag(const std::string& tag) const { @@ -207,6 +209,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to } } else if (toolParserName == "gemma4") { toolParser = std::make_unique(tokenizer); + } else if (toolParserName == "minicpm5") { + toolParser = std::make_unique(tokenizer, toolNameSchemaMap); } else if (!toolParserName.empty()) { throw std::runtime_error("Unsupported tool parser: \"" + toolParserName + "\". Supported tool parsers are: " + getSupportedToolParserNamesAsString()); @@ -218,6 +222,8 @@ OutputParser::OutputParser(ov::genai::Tokenizer& tokenizer, const std::string to reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "gptoss") { reasoningParser = std::make_unique(tokenizer); + } else if (reasoningParserName == "minicpm5") { + reasoningParser = std::make_unique(tokenizer); } else if (reasoningParserName == "lfm2") { reasoningParser = std::make_unique(tokenizer); } else if (!reasoningParserName.empty()) { diff --git a/src/llm/io_processing/parser_config_validation.cpp b/src/llm/io_processing/parser_config_validation.cpp index cba275f7e5..770993cd1b 100644 --- a/src/llm/io_processing/parser_config_validation.cpp +++ b/src/llm/io_processing/parser_config_validation.cpp @@ -32,6 +32,7 @@ const std::vector& getSupportedToolParserNames() { "devstral", "lfm2", "gemma4", + "minicpm5", }; return names; } @@ -41,6 +42,7 @@ const std::vector& getSupportedReasoningParserNames() { "qwen3", "gemma4", "gptoss", + "minicpm5", "lfm2", }; return names; diff --git a/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp b/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp index 4f38be2548..50476c1e89 100644 --- a/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp +++ b/src/llm/io_processing/qwen3coder/qwen3coder_tool_parser.cpp @@ -36,21 +36,6 @@ const std::string Qwen3CoderToolParser::PARAMETER_END_TAG = ""; const std::string Qwen3CoderToolParser::FUNCTION_END_TAG = ""; const std::string Qwen3CoderToolParser::TOOL_END_TAG = ""; -static void trimNewline(std::string& str) { - if (str.empty()) { - return; - } - if (str.back() == '\n') { - str.pop_back(); - } - if (str.empty()) { - return; - } - if (str.front() == '\n') { - str.erase(str.begin()); - } -} - Status Qwen3CoderToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& outContent) { if (toolCallPositions.begin.size() != toolCallPositions.end.size()) { SPDLOG_DEBUG("Mismatched tool tags, begin: {}, end: {}", toolCallPositions.begin.size(), toolCallPositions.end.size()); @@ -67,53 +52,6 @@ Status Qwen3CoderToolParserImpl::removeToolCallsFromContentIfNeeded(std::string& } return StatusCode::OK; } -// Exemplary schemas -// {"type":"object","properties":{"location":{"type":"string"},"provide_temperature":{"type":"boolean"}},"required":["location"]} -// {"type":"object","required":["location"],"properties":{"location":{"type":"string","description":"The location for which to get the weather, in the format of 'City, State', such as 'San Francisco, CA' if State for the city exists. 'City, Country' if State for the city doesn't exist."},"unit":{"type":"string","description":"The unit of temperature for the weather report.","enum":["celsius","fahrenheit"],"default":"fahrenheit"}}} -static const ParametersTypeMap_t parseToolSchema(const std::string& functionName, const rapidjson::Value& schema) { - // we want to create mapping of parameter name to parameter type - SPDLOG_TRACE("Parse tool schema for tool: {}, schema: {}", functionName, schema.GetString()); - ParametersTypeMap_t result; - if (!schema.IsObject()) { - SPDLOG_DEBUG("Tool schema is not a JSON object for tool: {}, schema: {}", functionName, schema.GetString()); - return result; - } - if (!schema.HasMember("properties") || !schema["properties"].IsObject()) { - SPDLOG_DEBUG("Tool schema does not have properties object for tool: {}, schema: {}", functionName, schema.GetString()); - return result; - } - const rapidjson::Value& properties = schema["properties"]; - for (auto it = properties.MemberBegin(); it != properties.MemberEnd(); ++it) { - if (!it->value.IsObject()) { - SPDLOG_DEBUG("Tool schema property: {} is not an object for tool: {}, schema: {}", it->name.GetString(), functionName, schema.GetString()); - continue; - } - if (!it->value.HasMember("type") || !it->value["type"].IsString()) { - SPDLOG_DEBUG("Tool schema property: {} does not have type string for tool: {}, schema: {}", it->name.GetString(), functionName, schema.GetString()); - continue; - } - std::string paramName = it->name.GetString(); - std::string typeStr = it->value["type"].GetString(); - ParameterType type = ParameterType::UNKNOWN; - if (typeStr == "string") { - type = ParameterType::STRING; - } else if (typeStr == "number" || typeStr == "integer") { - type = ParameterType::NUMBER; - } else if (typeStr == "boolean") { - type = ParameterType::BOOLEAN; - } else if (typeStr == "array") { - type = ParameterType::ARRAY; - } else if (typeStr == "object") { - type = ParameterType::OBJECT; - } else { - SPDLOG_DEBUG("Tool schema property: {} has unknown type: {} for tool: {}, schema: {}", paramName, typeStr, functionName, schema.GetString()); - } - SPDLOG_TRACE("Tool:{} param:{} type:{}", functionName, paramName, typeStr); - result.emplace(paramName, type); - } - return result; -} - #define DEFINE_TAG_POSITION_AND_BREAK_IF_NOT_FOUND(TAG) \ auto pos = this->streamContent.find(TAG, this->getLastProcessedPosition()); \ if (pos == std::string::npos) { \ @@ -121,39 +59,6 @@ static const ParametersTypeMap_t parseToolSchema(const std::string& functionName break; \ } -static const char* jsonTypeOf(const rapidjson::Value& val) { - if (val.IsObject()) - return "object"; - if (val.IsArray()) - return "array"; - if (val.IsString()) - return "string"; - if (val.IsBool()) - return "bool"; - if (val.IsInt()) - return "int"; - if (val.IsUint()) - return "uint"; - if (val.IsInt64()) - return "int64"; - if (val.IsUint64()) - return "uint64"; - if (val.IsDouble()) - return "double"; - if (val.IsNumber()) - return "number"; - if (val.IsNull()) - return "null"; - return "unknown"; -} -static void enforceStringValue(rapidjson::Value& v, - rapidjson::Document::AllocatorType& alloc) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - v.Accept(writer); - v.SetString(buffer.GetString(), buffer.GetLength(), alloc); -} - void Qwen3CoderToolParserImpl::addParameterToCurrentFunctionDoc(std::string& parameterValueAsString) { if (this->removeNewlineAroundParameters) trimNewline(parameterValueAsString); @@ -167,11 +72,7 @@ void Qwen3CoderToolParserImpl::addParameterToCurrentFunctionDoc(std::string& par if (paramIt != this->toolsParametersTypeMap.end()) { auto paramJt = paramIt->second.find(currentParameterName); if (paramJt != paramIt->second.end() && (paramJt->second == ParameterType::BOOLEAN)) { - if (parameterValueAsString == "True" || parameterValueAsString == "TRUE") { - parameterValueAsString = "true"; - } else if (parameterValueAsString == "False" || parameterValueAsString == "FALSE") { - parameterValueAsString = "false"; - } + normalizeBooleanString(parameterValueAsString); } } temp.Parse(parameterValueAsString.c_str()); @@ -312,18 +213,6 @@ std::optional Qwen3CoderToolParserImpl::parseChunk(const std::strin return std::nullopt; } -static ToolsParameterTypeMap_t createToolsParametersTypesMap(const ToolsSchemas_t& toolsSchemas) { - SPDLOG_TRACE("Creating tools parameters types map with schemas size: {}", toolsSchemas.size()); - ToolsParameterTypeMap_t toolsParametersTypes; - for (const auto& [toolName, toolSchemaWrapper] : toolsSchemas) { - const auto& toolSchemaStringRepr = toolSchemaWrapper.stringRepr; - const auto& toolSchemaRapidjsonRepr = toolSchemaWrapper.rapidjsonRepr; - SPDLOG_TRACE("Creating tools parameters types for tool: {}, schema: {}", toolName, toolSchemaStringRepr); - toolsParametersTypes.emplace(toolName, parseToolSchema(toolName, *toolSchemaRapidjsonRepr)); - } - return toolsParametersTypes; -} - void Qwen3CoderToolParser::lazyFillInitToolParametersTypesMap() { if (this->filledParametersTypesMap) { return; diff --git a/src/llm/io_processing/utils.cpp b/src/llm/io_processing/utils.cpp index c58ed1ccf0..7c46d1a0a9 100644 --- a/src/llm/io_processing/utils.cpp +++ b/src/llm/io_processing/utils.cpp @@ -91,4 +91,89 @@ size_t findInStringRespectingSpecialChars(const std::string& str, const std::str } return std::string::npos; } + +void trimNewline(std::string& str) { + if (str.empty()) { + return; + } + if (str.back() == '\n') { + str.pop_back(); + } + if (str.empty()) { + return; + } + if (str.front() == '\n') { + str.erase(str.begin()); + } +} + +const char* jsonTypeOf(const rapidjson::Value& val) { + if (val.IsObject()) + return "object"; + if (val.IsArray()) + return "array"; + if (val.IsString()) + return "string"; + if (val.IsBool()) + return "bool"; + if (val.IsInt()) + return "int"; + if (val.IsUint()) + return "uint"; + if (val.IsInt64()) + return "int64"; + if (val.IsUint64()) + return "uint64"; + if (val.IsDouble()) + return "double"; + if (val.IsNumber()) + return "number"; + if (val.IsNull()) + return "null"; + return "unknown"; +} + +void enforceStringValue(rapidjson::Value& v, rapidjson::Document::AllocatorType& alloc) { + if (v.IsString()) { + return; + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + v.Accept(writer); + v.SetString(buffer.GetString(), buffer.GetLength(), alloc); +} + +void normalizeBooleanString(std::string& value) { + if (value == "True" || value == "TRUE") { + value = "true"; + } else if (value == "False" || value == "FALSE") { + value = "false"; + } +} + +std::string replaceSingleWithDoubleQuotes(const std::string& input) { + std::string result; + result.reserve(input.size()); + bool insideDoubleQuote = false; + bool insideSingleQuote = false; + for (size_t i = 0; i < input.size(); ++i) { + char c = input[i]; + if (c == '\\' && i + 1 < input.size()) { + result += c; + result += input[++i]; + continue; + } + if (c == '"' && !insideSingleQuote) { + insideDoubleQuote = !insideDoubleQuote; + result += c; + } else if (c == '\'' && !insideDoubleQuote) { + insideSingleQuote = !insideSingleQuote; + result += '"'; + } else { + result += c; + } + } + return result; +} + } // namespace ovms diff --git a/src/llm/io_processing/utils.hpp b/src/llm/io_processing/utils.hpp index 79c7358af9..4568ec0851 100644 --- a/src/llm/io_processing/utils.hpp +++ b/src/llm/io_processing/utils.hpp @@ -24,11 +24,27 @@ #pragma warning(pop) namespace ovms { -size_t findInStringRespectingSpecialChars(const std::string& str, const std::string& target, size_t startPos); -void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer); // Generates random alphanumeric string of length 9 for tool call ID std::string generateRandomId(); size_t findInStringRespectingSpecialChars(const std::string& str, const std::string& target, size_t startPos); void writeArgumentOfAnyType(const rapidjson::Value& arg, rapidjson::Writer& writer); + +// ---- Tool parser helpers shared between attribute/tag style parsers (e.g. qwen3coder, minicpm5) ---- + +// Trims a single leading and a single trailing '\n' from str (in place). +void trimNewline(std::string& str); + +// Returns a human-readable name of the JSON value type (for tracing). +const char* jsonTypeOf(const rapidjson::Value& val); + +// Re-serializes a JSON value and stores it back as a JSON string value. +void enforceStringValue(rapidjson::Value& v, rapidjson::Document::AllocatorType& alloc); + +// Normalizes Python-style booleans ("True"/"TRUE" -> "true", "False"/"FALSE" -> "false") in place. +void normalizeBooleanString(std::string& value); + +// Replaces single-quote string delimiters with double quotes for JSON compatibility. +// Handles nested quoting: apostrophes inside double-quoted strings are preserved. +std::string replaceSingleWithDoubleQuotes(const std::string& input); } // namespace ovms diff --git a/src/test/llm/chat_template_analyzer_test.cpp b/src/test/llm/chat_template_analyzer_test.cpp index 56f18f5458..6ecbf7e14b 100644 --- a/src/test/llm/chat_template_analyzer_test.cpp +++ b/src/test/llm/chat_template_analyzer_test.cpp @@ -191,6 +191,19 @@ TEST_F(ChatTemplateAnalyzerTest, unknownTemplateReturnsEmpty) { EXPECT_FALSE(result.caps.supportsToolCalls); } +// --- MiniCPM5 --- + +TEST_F(ChatTemplateAnalyzerTest, detectsMinicpm5) { + std::string tmpl = loadTemplate("chat_template_minicpm5.jinja"); + ASSERT_FALSE(tmpl.empty()); + auto result = ChatTemplateAnalyzer::analyze(tmpl); + ASSERT_TRUE(result.detectedToolParser.has_value()); + EXPECT_EQ(result.detectedToolParser.value(), "minicpm5"); + ASSERT_TRUE(result.detectedReasoningParser.has_value()); + EXPECT_EQ(result.detectedReasoningParser.value(), "minicpm5"); + EXPECT_TRUE(result.caps.supportsToolCalls); +} + // --- Priority: Devstral over Mistral (inline — tests specific precedence logic) --- TEST_F(ChatTemplateAnalyzerTest, devstralTakesPriorityOverMistral) { diff --git a/src/test/llm/chat_template_end_to_end_jinja_test.cpp b/src/test/llm/chat_template_end_to_end_jinja_test.cpp index 50d0130448..3d284ea300 100644 --- a/src/test/llm/chat_template_end_to_end_jinja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_jinja_test.cpp @@ -630,3 +630,38 @@ What's the weather in Paris?<|im_end|> )"; EXPECT_EQ(appliedOutput, expectedOutput); } + +// ============================================================================= +// MiniCPM5 uses ... format. +// Template uses from_json filter which is supported by Jinja. +// ============================================================================= +TEST_F(ChatTemplateEndToEndJinjaTest, MiniCPM5_ToolCallWithStringArgs) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_minicpm5.jinja"); + ASSERT_FALSE(chatTemplate.empty()); + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(); + + ASSERT_FALSE(exceptionThrownDuringApplication); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "minicpm5"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "minicpm5"); + + EXPECT_TRUE(caps.supportsToolCalls); + EXPECT_TRUE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); + + std::string expectedOutput = R"(<|im_start|>user +What's the weather in Paris?<|im_end|> +<|im_start|>assistant +Pariscelsius<|im_end|> +<|im_start|>assistant +)"; + EXPECT_EQ(appliedOutput, expectedOutput); +} diff --git a/src/test/llm/chat_template_end_to_end_minja_test.cpp b/src/test/llm/chat_template_end_to_end_minja_test.cpp index 1e068e4c5a..c271788a1a 100644 --- a/src/test/llm/chat_template_end_to_end_minja_test.cpp +++ b/src/test/llm/chat_template_end_to_end_minja_test.cpp @@ -596,6 +596,30 @@ What's the weather in Paris?<|im_end|> EXPECT_EQ(appliedOutput, expectedOutput); } +TEST_F(ChatTemplateEndToEndMinjaTest, MiniCPM5_ToolCallWithStringArgsExpectedToFailMinjaDontSupportThisTemplate) { + chatTemplate = loadTemplateFile(chatTemplatesPath + "/chat_template_minicpm5.jinja"); + ASSERT_FALSE(chatTemplate.empty()) << "Failed to load minicpm5 template"; + + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"user","content":"What's the weather in Paris?"})")); + chatHistory.push_back(ov::genai::JsonContainer::from_json_string( + R"({"role":"assistant","content":"","tool_calls":[{"id":"call_abc123","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Paris\",\"unit\":\"celsius\"}"}}]})")); + + run(true); + + ASSERT_FALSE(exceptionThrownDuringApplication); + ASSERT_TRUE(basicRenderOk); + + ASSERT_TRUE(analysisResult.detectedToolParser.has_value()); + EXPECT_EQ(analysisResult.detectedToolParser.value(), "minicpm5"); + ASSERT_TRUE(analysisResult.detectedReasoningParser.has_value()); + EXPECT_EQ(analysisResult.detectedReasoningParser.value(), "minicpm5"); + + EXPECT_FALSE(caps.supportsToolCalls); + EXPECT_FALSE(caps.requiresObjectArguments); + EXPECT_TRUE(caps.missnamedReasoningField.empty()); +} + // ============================================================================= // Synthetic test: template that throws on basic rendering (e.g. uses undefined // filter). The basic render probe should catch this and return false. diff --git a/src/test/llm/chat_templates/chat_template_minicpm5.jinja b/src/test/llm/chat_templates/chat_template_minicpm5.jinja new file mode 100755 index 0000000000..c514ff342a --- /dev/null +++ b/src/test/llm/chat_templates/chat_template_minicpm5.jinja @@ -0,0 +1,158 @@ +{{- bos_token }} +{%- macro render_tool(tool_call) -%} + {{- '' }} + {%- if tool_call.arguments %} + {%- set args = tool_call.arguments %} + {% if args is string %} + {{- args }} + {% else %} + {%- for param_name, param_value in args.items() %} + {{- '' }} + {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %} + {{- '' }} + {%- else %} + {{- param_value }} + {%- endif %} + {{- '' }} + {%- endfor %} + {% endif %} + {%- endif %} + {{- '' }} +{%- endmacro %} + +{%- if tools %} + {%- set tool_definitions %} + {{- "# Tools\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n\n\nTool usage guidelines:\n- You may call zero or more functions. If no function calls are needed, just answer normally and do not include any .\n- When calling a function, return an XML object within using:\nparam-value\n- param-value may be multi-line. If it contains <, & or newline characters, wrap it in a CDATA block: ' }} + {%- endset %} + + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if '' in messages[0].content %} + {{- messages[0].content.replace('', tool_definitions) }} + {%- else %} + {{- messages[0].content + '\n\n' + tool_definitions }} + {%- endif %} + {%- else %} + {{- tool_definitions.lstrip() }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + + {%- if message.tool_calls %} + {%- set content_parts = content.split('') %} + {%- set processed_content = content_parts[0] %} + {%- set tool_calls_count = message.tool_calls|length %} + {%- set tool_sep_count = content_parts|length - 1 %} + {%- set min_count = [tool_calls_count, tool_sep_count]|min %} + + {%- for i in range(1, content_parts|length) %} + {%- set tool_index = i - 1 %} + {%- if tool_index < tool_calls_count %} + {%- set tool_call = message.tool_calls[tool_index] %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- set single_tool_xml %}{{- render_tool(tool_call) }}{%- endset %} + {%- set processed_content = processed_content + single_tool_xml + content_parts[i] %} + {%- else %} + {%- set processed_content = processed_content + content_parts[i] %} + {%- endif %} + {%- endfor %} + + {%- if tool_calls_count > tool_sep_count %} + {%- for remaining_index in range(tool_sep_count, tool_calls_count) %} + {%- set tool_call = message.tool_calls[remaining_index] %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- set remaining_tool_xml %}{{- render_tool(tool_call) }}{%- endset %} + {%- set processed_content = processed_content + remaining_tool_xml %} + {%- endfor %} + {%- endif %} + + {%- set content = processed_content %} + {%- endif %} + + {%- if loop.index0 > ns.last_query_index %} + {%- if reasoning_content %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + + {%- if message.tool_calls and not has_tool_sep %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- render_tool(tool_call) }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {%- if message.content is string %} + {{- content }} + {%- else %} + {{- message.content | tojson }} + {%- endif %} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined %} + {%- if enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- elif enable_thinking is true %} + {{- '\n' }} + {%- endif %} + {%- endif %} +{%- endif %} diff --git a/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp b/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp new file mode 100644 index 0000000000..49b9576008 --- /dev/null +++ b/src/test/llm/output_parsers/minicpm5_output_parser_test.cpp @@ -0,0 +1,873 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include "src/llm/io_processing/base_output_parser.hpp" +#include "src/llm/io_processing/output_parser.hpp" +#include "src/llm/io_processing/minicpm5/minicpm5_tool_parser.hpp" +#include "src/llm/io_processing/minicpm5/minicpm5_reasoning_parser.hpp" +#include "src/test/platform_utils.hpp" + +using namespace ovms; + +#ifdef _WIN32 +const std::string minicpm5TokenizerPath = getWindowsRepoRootPath() + "\\src\\test\\llm_testing\\openbmb\\MiniCPM5-1B"; +#else +const std::string minicpm5TokenizerPath = "/ovms/src/test/llm_testing/openbmb/MiniCPM5-1B"; +#endif + +using ovms::ParameterType; +using ovms::ToolsParameterTypeMap_t; + +static std::unique_ptr minicpm5Tokenizer; + +// Schemas used to drive type-aware parameter insertion +static std::map minicpm5ToolSchemasInput = { + {"get_weather", R"({"properties":{"city":{"type":"string"},"unit":{"type":"string"}},"required":["city"]})"}, + {"calculate", R"({"properties":{"a":{"type":"integer"},"b":{"type":"integer"}},"required":["a","b"]})"}, + {"search", R"({"properties":{"query":{"type":"string"}},"required":["query"]})"}, + {"get_stock", R"({"properties":{"ticker":{"type":"string"},"count":{"type":"integer"}},"required":["ticker"]})"}, + {"dummy", R"({"properties":{"config":{"type":"object"}},"required":["config"]})"}, + {"pwd", R"({"properties":{},"required":[]})"}, +}; + +static std::vector> minicpm5SchemaDocsStorage; + +static ToolsSchemas_t convertToToolsSchemas(const std::map& input) { + ToolsSchemas_t result; + minicpm5SchemaDocsStorage.clear(); + for (const auto& [name, schemaStr] : input) { + auto schemaDoc = std::make_unique(); + if (schemaDoc->Parse(schemaStr.c_str()).HasParseError()) { + throw std::runtime_error("Failed to parse schema for tool: " + name); + } + result[name] = {schemaDoc.get(), schemaStr}; + minicpm5SchemaDocsStorage.push_back(std::move(schemaDoc)); + } + return result; +} + +static ovms::ToolsSchemas_t minicpm5ToolsSchemas = convertToToolsSchemas(minicpm5ToolSchemasInput); + +// Type map used when testing Minicpm5ToolParserImpl directly (without going through OutputParser) +static ToolsParameterTypeMap_t minicpm5TypeMap = { + {"get_weather", {{"city", ParameterType::STRING}, {"unit", ParameterType::STRING}}}, + {"calculate", {{"a", ParameterType::NUMBER}, {"b", ParameterType::NUMBER}}}, + {"search", {{"query", ParameterType::STRING}}}, + {"get_stock", {{"ticker", ParameterType::STRING}, {"count", ParameterType::NUMBER}}}, + {"dummy", {{"config", ParameterType::OBJECT}}}, +}; + +static std::string jsonValueToString(const rapidjson::Value& val) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + val.Accept(writer); + return buffer.GetString(); +} + +// ---- Test fixture ---- + +class Minicpm5OutputParserTest : public ::testing::Test { +protected: + std::unique_ptr outputParser; + + static void SetUpTestSuite() { + try { + minicpm5Tokenizer = std::make_unique(minicpm5TokenizerPath); + } catch (const std::exception& e) { + FAIL() << "Failed to initialize tokenizer: " << e.what(); + } catch (...) { + FAIL() << "Failed to initialize tokenizer due to unknown error."; + } + } + + static void TearDownTestSuite() { + minicpm5Tokenizer.reset(); + } + + void SetUp() override { + outputParser = std::make_unique(*minicpm5Tokenizer, "minicpm5", "minicpm5", minicpm5ToolsSchemas); + } + + std::vector encodeInput(const std::string& input) { + if (input == Minicpm5ReasoningParser::reasoningStartTag) { + return {Minicpm5ReasoningParser::reasoningStartTokenId}; + } + if (input == Minicpm5ReasoningParser::reasoningEndTag) { + return {Minicpm5ReasoningParser::reasoningEndTokenId}; + } + auto generatedTensor = minicpm5Tokenizer->encode(input, ov::genai::add_special_tokens(true)).input_ids; + return std::vector( + generatedTensor.data(), + generatedTensor.data() + generatedTensor.get_size()); + } + + ParsedOutput generateParsedOutput(const std::string& input) { + auto generatedTokens = encodeInput(input); + return outputParser->parse(generatedTokens, true); + } + + void assertReasoningVec(const std::vector>>& chunkToDeltaVec) { + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + std::vector tokens = {}; + if (chunk == Minicpm5ReasoningParser::reasoningStartTag) { + tokens = {Minicpm5ReasoningParser::reasoningStartTokenId}; + } else if (chunk == Minicpm5ReasoningParser::reasoningEndTag) { + tokens = {Minicpm5ReasoningParser::reasoningEndTokenId}; + } else { + tokens = encodeInput(chunk); + } + std::optional doc = outputParser->parseChunk(chunk, tokens, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + EXPECT_EQ(docStr, expectedDelta.value()) << "Mismatch for chunk: " << chunk; + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } + } + + void assertStreamingVec(const std::vector>>& chunkToDeltaVec) { + for (const auto& [chunk, finishReason, expectedDelta] : chunkToDeltaVec) { + auto tokens = encodeInput(chunk); + std::optional doc = outputParser->parseChunk(chunk, tokens, true, finishReason); + if (!expectedDelta.has_value() && !doc.has_value()) { + continue; // Both are nullopt, OK + } + if (expectedDelta.has_value() && doc.has_value()) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + std::string docStr = buffer.GetString(); + std::string expected = expectedDelta.value(); + std::string idKey = "\"id\":\""; + auto docIdPos = docStr.find(idKey); + auto expectedIdPos = expected.find(idKey); + if (docIdPos != std::string::npos && expectedIdPos != std::string::npos) { + auto docIdStart = docIdPos + idKey.size(); + auto docIdEnd = docStr.find("\"", docIdStart); + auto expectedIdStart = expectedIdPos + idKey.size(); + auto expectedIdEnd = expected.find("\"", expectedIdStart); + ASSERT_NE(docIdEnd, std::string::npos); + ASSERT_NE(expectedIdEnd, std::string::npos); + std::string docId = docStr.substr(docIdStart, docIdEnd - docIdStart); + std::string expectedId = expected.substr(expectedIdStart, expectedIdEnd - expectedIdStart); + EXPECT_EQ(docId.size(), expectedId.size()) << "ID length mismatch for chunk: " << chunk; + EXPECT_TRUE(std::all_of(docId.begin(), docId.end(), ::isalnum)) << "ID not alphanumeric for chunk: " << chunk; + // Compare everything except the id value + std::string docStrNoId = docStr; + std::string expectedNoId = expected; + docStrNoId.replace(docIdStart, docId.size(), std::string(docId.size(), '*')); + expectedNoId.replace(expectedIdStart, expectedId.size(), std::string(expectedId.size(), '*')); + EXPECT_EQ(docStrNoId, expectedNoId) << "Mismatch for chunk (ignoring id value): " << chunk; + } else { + EXPECT_EQ(docStr, expected) << "Mismatch for chunk: " << chunk; + } + } else { + std::string expectedStr = expectedDelta.has_value() ? expectedDelta.value() : "std::nullopt"; + std::string docStr = doc.has_value() ? [&]() { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc->Accept(writer); + return std::string(buffer.GetString()); + }() + : "std::nullopt"; + FAIL() << "Mismatch between expectedDelta and doc for chunk: " << chunk + << "\nexpectedDelta: " << expectedStr + << "\ndoc: " << docStr; + } + } + } +}; + +// ---- Unary parse tests ---- + +TEST_F(Minicpm5OutputParserTest, ParseSingleFunctionCallTwoParams) { + const std::string input = + R"(Beijingcelsius)"; + ParsedOutput parsedOutput = generateParsedOutput(input); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1u); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city":"Beijing","unit":"celsius"})"); + EXPECT_FALSE(parsedOutput.toolCalls[0].id.empty()); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); +} + +TEST_F(Minicpm5OutputParserTest, ParseMultipleFunctionCalls) { + const std::string input = + R"(Tokyo)" + R"(OpenVINO)"; + ParsedOutput parsedOutput = generateParsedOutput(input); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 2u); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city":"Tokyo"})"); + EXPECT_EQ(parsedOutput.toolCalls[1].name, "search"); + EXPECT_EQ(parsedOutput.toolCalls[1].arguments, R"({"query":"OpenVINO"})"); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); +} + +TEST_F(Minicpm5OutputParserTest, ParseIntegerTypedParam) { + const std::string input = + R"(73)"; + ParsedOutput parsedOutput = generateParsedOutput(input); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1u); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "calculate"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"a":7,"b":3})"); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); +} + +TEST_F(Minicpm5OutputParserTest, ParseMixedStringAndIntegerParams) { + const std::string input = + R"(INTC42)"; + ParsedOutput parsedOutput = generateParsedOutput(input); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1u); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_stock"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"ticker":"INTC","count":42})"); + EXPECT_EQ(parsedOutput.content, ""); + EXPECT_EQ(parsedOutput.reasoning, ""); +} + +// This scenario will be handled only in unary, in streaming it's not possible to parse reasoning without the starting tag +TEST_F(Minicpm5OutputParserTest, ParseReasoningWithoutStartingTag) { + const std::string input = "This is my internal reasoning about what to call."; + ParsedOutput parsedOutput = generateParsedOutput(input); + + EXPECT_EQ(parsedOutput.toolCalls.size(), 0u); + EXPECT_NE(parsedOutput.reasoning.find("internal reasoning"), std::string::npos); + EXPECT_EQ(parsedOutput.content, ""); +} + +TEST_F(Minicpm5OutputParserTest, ParseWithThinkBlockHandledByReasoningParser) { + constexpr int64_t thinkStartTokenId = Minicpm5ReasoningParser::reasoningStartTokenId; + constexpr int64_t thinkEndTokenId = Minicpm5ReasoningParser::reasoningEndTokenId; + + auto outputParserWithReasoning = + std::make_unique(*minicpm5Tokenizer, "minicpm5", "minicpm5", minicpm5ToolsSchemas); + + auto encode = [](ov::genai::Tokenizer& tok, const std::string& text) { + auto tensor = tok.encode(text, ov::genai::add_special_tokens(false)).input_ids; + return std::vector(tensor.data(), tensor.data() + tensor.get_size()); + }; + + std::vector generatedTokens; + generatedTokens.push_back(thinkStartTokenId); + auto reasoningTokens = encode(*minicpm5Tokenizer, "This is my internal reasoning about what to call."); + generatedTokens.insert(generatedTokens.end(), reasoningTokens.begin(), reasoningTokens.end()); + generatedTokens.push_back(thinkEndTokenId); + auto functionCallTokens = encode(*minicpm5Tokenizer, R"(Intel)"); + generatedTokens.insert(generatedTokens.end(), functionCallTokens.begin(), functionCallTokens.end()); + + ParsedOutput parsedOutput = outputParserWithReasoning->parse(generatedTokens, true); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1u); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "search"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"query":"Intel"})"); + EXPECT_NE(parsedOutput.reasoning.find("internal reasoning"), std::string::npos); + EXPECT_EQ(parsedOutput.content.find(""), std::string::npos); + EXPECT_EQ(parsedOutput.content, ""); +} + +TEST_F(Minicpm5OutputParserTest, RequiresStreamingWithSpecialTokens) { + Minicpm5ToolParser toolParser(*minicpm5Tokenizer, minicpm5ToolsSchemas); + EXPECT_TRUE(toolParser.requiresStreamingWithSpecialTokens()); + Minicpm5ReasoningParser reasoningParser(*minicpm5Tokenizer); + EXPECT_TRUE(reasoningParser.requiresStreamingWithSpecialTokens()); + EXPECT_NO_THROW({ + OutputParser parser(*minicpm5Tokenizer, "minicpm5", "minicpm5", minicpm5ToolsSchemas); + (void)parser; + }); +} + +TEST_F(Minicpm5OutputParserTest, ParseNoToolCalls) { + const std::string input = "Sure, here is the answer to your question."; + ParsedOutput parsedOutput = generateParsedOutput(input); + + EXPECT_EQ(parsedOutput.toolCalls.size(), 0u); + EXPECT_NE(parsedOutput.content.find("Sure"), std::string::npos); +} + +TEST_F(Minicpm5OutputParserTest, ParseContentAroundToolCall) { + const std::string input = + "Let me check the weather. " + R"(Paris)" + " Done."; + ParsedOutput parsedOutput = generateParsedOutput(input); + + ASSERT_EQ(parsedOutput.toolCalls.size(), 1u); + EXPECT_EQ(parsedOutput.toolCalls[0].name, "get_weather"); + EXPECT_EQ(parsedOutput.toolCalls[0].arguments, R"({"city":"Paris"})"); + EXPECT_EQ(parsedOutput.content.find(""), std::string::npos); + EXPECT_NE(parsedOutput.content, "Let me check the weather. Done."); +} + +TEST(Minicpm5ToolParserImplTest, SingleCallNoContent) { + const std::string input = + R"(Berlin)"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, R"({"city":"Berlin"})"); + EXPECT_EQ(parser.getCurrentState(), Minicpm5ToolParserImpl::State::Content); + EXPECT_FALSE(calls[0].id.empty()); +} + +TEST(Minicpm5ToolParserImplTest, SingleCallWithEmptyParam) { + const std::string input = + R"()"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "dummy"); + EXPECT_EQ(calls[0].arguments, R"({"config":""})"); +} + +TEST(Minicpm5ToolParserImplTest, SingleCallWithNoArgs) { + const std::string input = + R"()"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "dummy"); + EXPECT_EQ(calls[0].arguments, R"({})"); +} + +TEST(Minicpm5ToolParserImplTest, TwoCalls) { + const std::string input = + R"(Rome)" + R"(1020)"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 2u); + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, R"({"city":"Rome"})"); + EXPECT_EQ(calls[1].name, "calculate"); + EXPECT_EQ(calls[1].arguments, R"({"a":10,"b":20})"); +} + +TEST(Minicpm5ToolParserImplTest, ThinkBlockTreatedAsContent) { + const std::string input = + "Reasoning here." + R"(test)"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "search"); + EXPECT_EQ(calls[0].arguments, R"({"query":"test"})"); +} + +TEST(Minicpm5ToolParserImplTest, IntegerParam) { + const std::string input = + R"(515)"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].arguments, R"({"a":5,"b":15})"); +} + +TEST(Minicpm5ToolParserImplTest, NoToolCalls) { + const std::string input = "Just some content without any function calls."; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + EXPECT_FALSE(callsOpt.has_value()); + EXPECT_EQ(parser.getCurrentState(), Minicpm5ToolParserImpl::State::Content); + EXPECT_EQ(parser.getLastProcessedPosition(), 0u); +} + +TEST(Minicpm5ToolParserImplTest, NewlinesAroundParamValue) { + const std::string input = + "\nBeijing\n"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + EXPECT_EQ(callsOpt.value()[0].arguments, R"({"city":"Beijing"})"); +} + +TEST(Minicpm5ToolParserImplTest, RemoveToolCallsFromContent) { + const std::string input = + "Before. " + R"(intel)" + " After."; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto status = parser.removeToolCallsFromContentIfNeeded(content); + EXPECT_TRUE(status.ok()) << status.string(); + EXPECT_EQ(content.find(""), std::string::npos); +} + +static ToolCalls_t streamInFragments(const std::string& input, size_t fragmentSize, + const ToolsParameterTypeMap_t& typeMap) { + Minicpm5ToolParserImpl parser(typeMap); + ToolCalls_t collected; + for (size_t i = 0; i < input.size(); i += fragmentSize) { + std::string chunk = input.substr(i, fragmentSize); + auto callsOpt = parser.parseChunk(chunk); + if (callsOpt.has_value()) { + for (auto& c : callsOpt.value()) { + collected.push_back(c); + } + } + } + return collected; +} + +TEST(Minicpm5ToolParserImplStreamTest, CharByCharSingleCall) { + const std::string input = + R"(Beijingcelsius)"; + auto calls = streamInFragments(input, 1, minicpm5TypeMap); // one character per chunk + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, R"({"city":"Beijing","unit":"celsius"})"); +} + +TEST(Minicpm5ToolParserImplStreamTest, VariousFragmentSizesSingleCall) { + const std::string input = + R"(Beijing)"; + for (size_t frag = 1; frag <= input.size(); ++frag) { + auto calls = streamInFragments(input, frag, minicpm5TypeMap); + ASSERT_EQ(calls.size(), 1u) << "fragment size " << frag; + EXPECT_EQ(calls[0].name, "get_weather") << "fragment size " << frag; + EXPECT_EQ(calls[0].arguments, R"({"city":"Beijing"})") << "fragment size " << frag; + } +} + +TEST(Minicpm5ToolParserImplStreamTest, TwoCallsCharByChar) { + const std::string input = + R"(Rome)" + R"(1020)"; + auto calls = streamInFragments(input, 1, minicpm5TypeMap); + ASSERT_EQ(calls.size(), 2u); + EXPECT_EQ(calls[0].name, "get_weather"); + EXPECT_EQ(calls[0].arguments, R"({"city":"Rome"})"); + EXPECT_EQ(calls[1].name, "calculate"); + EXPECT_EQ(calls[1].arguments, R"({"a":10,"b":20})"); +} + +TEST(Minicpm5ToolParserImplStreamTest, ThinkBlockAndContentCharByChar) { + const std::string input = + R"(Let me check.I should call the toolIntel)"; + auto calls = streamInFragments(input, 1, minicpm5TypeMap); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "search"); + EXPECT_EQ(calls[0].arguments, R"({"query":"Intel"})"); +} + +TEST(Minicpm5ToolParserImplTest, UnterminatedFunctionNoCall) { + const std::string input = + R"(Berlin)"; // truncated mid-value + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + EXPECT_FALSE(callsOpt.has_value() && !callsOpt.value().empty()); +} + +TEST(Minicpm5ToolParserImplTest, EmptyParamValue) { + const std::string input = + R"()"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + ASSERT_EQ(callsOpt.value().size(), 1u); + EXPECT_EQ(callsOpt.value()[0].name, "get_weather"); + EXPECT_EQ(callsOpt.value()[0].arguments, R"({"city":""})"); +} + +TEST(Minicpm5ToolParserImplTest, AngleBracketInPlainTextNoCall) { + const std::string input = "The value of a < b is true, and 3 < 4 as well."; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + EXPECT_FALSE(callsOpt.has_value() && !callsOpt.value().empty()); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithToolCallAndReasoning) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"This", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"This"}})"}, + {" is", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" is"}})"}, + {" my", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" my"}})"}, + {" internal", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" internal"}})"}, + {" reasoning", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" reasoning"}})"}, + {" about", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" about"}})"}, + {" what", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" what"}})"}, + {" to", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" to"}})"}, + {" call.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" call."}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\"key\":\"value\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithToolCallAndSpecialTags) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\"key\":\"value\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + {"<|im_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +// It's possible that the model starts reasoning without the starting tag; in that case treat it as regular content. +TEST_F(Minicpm5OutputParserTest, StreamingWithReasoningWithoutStartingTag) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"This", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"This"}})"}, + {" is", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" is"}})"}, + {" my", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" my"}})"}, + {" internal", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" internal"}})"}, + {" reasoning", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" reasoning"}})"}, + {" about", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" about"}})"}, + {" what", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" what"}})"}, + {" to", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" to"}})"}, + {" call.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" call."}})"}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":""}})"}, + {"Some", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Some"}})"}, + {" content", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" content"}})"}, + {" after", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" after"}})"}, + {" the", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" the"}})"}, + {" reasoning.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" reasoning."}})"}, + {"<|im_end|>", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithToolCallReasoningAndContent) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"This", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"This"}})"}, + {" is", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" is"}})"}, + {" my", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" my"}})"}, + {" internal", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" internal"}})"}, + {" reasoning", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" reasoning"}})"}, + {" about", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" about"}})"}, + {" what", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" what"}})"}, + {" to", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" to"}})"}, + {" call.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" call."}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\"key\":\"value\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + {"SOME CONTENT,", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"IT SHOULDN'T APPEAR", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithToolCallAndContent) { + std::vector>> chunkToDeltaVec{ + {"SOME CONTENT", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"SOME CONTENT"}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\"key\":\"value\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + {"CONTENT SHOULDN'T BE RETURNED AFTER TOOL CALL", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithReasoningAndContent) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"This", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":"This"}})"}, + {" is", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" is"}})"}, + {" my", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" my"}})"}, + {" internal", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" internal"}})"}, + {" reasoning", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" reasoning"}})"}, + {" about", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" about"}})"}, + {" what", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" what"}})"}, + {" to", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" to"}})"}, + {" call.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"reasoning_content":" call."}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Here", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Here"}})"}, + {" is", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" is"}})"}, + {" some", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" some"}})"}, + {" content.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" content."}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithContentOnly) { + std::vector>> chunkToDeltaVec{ + {"Here", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":"Here"}})"}, + {" is", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" is"}})"}, + {" some", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" some"}})"}, + {" content.", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"content":" content."}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithMultipleToolCalls) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\"key\":\"value\"}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"Tokyo", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"celsius", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"city\":\"Tokyo\",\"unit\":\"celsius\"}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"What does it mean \"if __name__ == '__main__':\n\tdict = { \"key\": \"value\" }\"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":2,"function":{"arguments":"{\"query\":\"What does it mean \\\"if __name__ == '__main__':\\n\\tdict = { \\\"key\\\": \\\"value\\\" }\\\"\"}"}}]}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithToolCallWithArrayAndObjectParams) { + std::vector>> chunkToDeltaVec{ + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\'key\':\'value\'}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"[\'1\', \'2\', \'3\']", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"{\"verbose\":true}", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, std::nullopt}, + {"", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"values\":[\"1\",\"2\",\"3\"],\"options\":{\"verbose\":true}}"}}]}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST(Minicpm5ToolParserImplTest, StringParamWithQuotesAndBackslashes) { + const std::string input = + R"(say "hi" and a backslash \ here)"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + EXPECT_EQ(calls[0].name, "search"); + // arguments must be valid JSON + rapidjson::Document argsDoc; + argsDoc.Parse(calls[0].arguments.c_str()); + ASSERT_FALSE(argsDoc.HasParseError()) << "arguments not valid JSON: " << calls[0].arguments; + ASSERT_TRUE(argsDoc.HasMember("query")); + ASSERT_TRUE(argsDoc["query"].IsString()); + EXPECT_EQ(std::string(argsDoc["query"].GetString()), R"(say "hi" and a backslash \ here)"); +} + +TEST_F(Minicpm5OutputParserTest, StreamingWithToolCallWithBiggerChunks) { + std::vector>> chunkToDeltaVec{ + {"{\'key\':\'value\'}", ov::genai::GenerationFinishReason::NONE, R"({"delta":{"tool_calls":[{"id":"XXXXXXXXX","type":"function","index":0,"function":{"name":"dummy","arguments":"{\"config\":{\"key\":\"value\"}}"}}]}})"}, + }; + + assertStreamingVec(chunkToDeltaVec); +} + +TEST(Minicpm5ToolParserImplTest, StringParamWithCodeSnippet) { + const std::string code = + "def f(x):\n return {\"a\": x, \"b\": [1, 2, 3]} // comment"; + const std::string input = + R"()" + code + R"()"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + rapidjson::Document argsDoc; + argsDoc.Parse(calls[0].arguments.c_str()); + ASSERT_FALSE(argsDoc.HasParseError()) << "arguments not valid JSON: " << calls[0].arguments; + ASSERT_TRUE(argsDoc.HasMember("query")); + ASSERT_TRUE(argsDoc["query"].IsString()); + EXPECT_EQ(std::string(argsDoc["query"].GetString()), code); +} + +TEST(Minicpm5ToolParserImplTest, ObjectParamWithSingleQuotes) { + const std::string input = + R"({'nested':true,'n':42})"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + rapidjson::Document argsDoc; + argsDoc.Parse(calls[0].arguments.c_str()); + ASSERT_FALSE(argsDoc.HasParseError()) << "arguments not valid JSON: " << calls[0].arguments; + ASSERT_TRUE(argsDoc.HasMember("config")); + ASSERT_TRUE(argsDoc["config"].IsObject()); + EXPECT_EQ(jsonValueToString(argsDoc["config"]), R"({"nested":true,"n":42})"); +} + +TEST(Minicpm5ToolParserImplTest, ArrayParamWithSingleQuotes) { + const std::string input = + R"([1, 2, 3, 'four'])"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + rapidjson::Document argsDoc; + argsDoc.Parse(calls[0].arguments.c_str()); + ASSERT_FALSE(argsDoc.HasParseError()) << "arguments not valid JSON: " << calls[0].arguments; + ASSERT_TRUE(argsDoc.HasMember("config")); + ASSERT_TRUE(argsDoc["config"].IsArray()); + EXPECT_EQ(jsonValueToString(argsDoc["config"]), R"([1,2,3,"four"])"); +} + +TEST(Minicpm5ToolParserImplTest, StringParamWithSpecialChars) { + const std::string value = "tab\tand emoji \xF0\x9F\x98\x80 and slash / and quote \" end"; + const std::string input = + R"()" + value + R"()"; + auto content = input; + Minicpm5ToolParserImpl parser(minicpm5TypeMap); + auto callsOpt = parser.parseChunk(content); + ASSERT_TRUE(callsOpt.has_value()); + auto& calls = callsOpt.value(); + ASSERT_EQ(calls.size(), 1u); + rapidjson::Document argsDoc; + argsDoc.Parse(calls[0].arguments.c_str()); + ASSERT_FALSE(argsDoc.HasParseError()) << "arguments not valid JSON: " << calls[0].arguments; + ASSERT_TRUE(argsDoc.HasMember("query")); + ASSERT_TRUE(argsDoc["query"].IsString()); + EXPECT_EQ(std::string(argsDoc["query"].GetString()), value); +} diff --git a/windows_prepare_llm_models.bat b/windows_prepare_llm_models.bat index 48f13e61b8..5c9554bc31 100644 --- a/windows_prepare_llm_models.bat +++ b/windows_prepare_llm_models.bat @@ -47,6 +47,7 @@ set "DEVSTRAL_MODEL=unsloth/Devstral-Small-2507" set "LFM2_MODEL=LiquidAI/LFM2-2.6B" set "LFM25_MODEL=LiquidAI/LFM2.5-8B-A1B" set "GEMMA4_MODEL=OpenVINO/gemma-4-E4B-it-int4-ov" +set "MINICPM5_MODEL=openbmb/MiniCPM5-1B" echo Downloading LLM testing models to directory %~1 set "PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu https://storage.openvinotoolkit.org/simple/wheels/nightly" @@ -88,6 +89,7 @@ call :download_tokenizer "%GPTOSS_MODEL%" "%~1\%GPTOSS_MODEL%" call :download_tokenizer "%DEVSTRAL_MODEL%" "%~1\%DEVSTRAL_MODEL%" call :download_tokenizer "%LFM2_MODEL%" "%~1\%LFM2_MODEL%" call :download_tokenizer "%LFM25_MODEL%" "%~1\%LFM25_MODEL%" +call :download_tokenizer "%MINICPM5_MODEL%" "%~1\%MINICPM5_MODEL%" call :download_openvino_tokenizer "%GEMMA4_MODEL%" "%~1" exit /b 0