From ff7385054181bb4b119e762cde4d44b471491b3e Mon Sep 17 00:00:00 2001 From: exzile Date: Sat, 27 Jun 2026 19:40:13 -0400 Subject: [PATCH 1/2] Release infer-request slot if start_async() throws in modelInferAsync In the async inference path (modelInferAsync, used by KServe gRPC and the C-API async endpoint), the ExecutingStreamIdGuard that holds an infer-request pool slot is std::move-d into the OV completion-callback lambda before inferRequest.start_async() is called. The slot is returned to OVInferRequestsQueue only when that lambda is destroyed, which normally happens when the async completion callback fires. If start_async() throws, the function returns immediately and the completion callback never fires, so the lambda (sole owner of the guard) is never destroyed and the infer-request slot is never returned to the pool. Under long-running concurrent load, repeated start_async failures slowly drain the pool until no idle infer request is available; requests then block and gRPC surfaces ResourceExhausted even though CPU/GPU/RAM are not saturated (matches the symptom class in #2871). Clear the callback in both start_async() catch blocks so the captured guard is destroyed and the slot is returned, mirroring the existing "set empty callback to release captures" pattern used after normal completion. The synchronous infer() path is unaffected (it uses a stack-allocated guard with RAII release on all paths). Builds clean (//src:ovms //src:ovms_test); CAPIInference.AsyncErrorHandling passes. The change is confined to the start_async error path. Signed-off-by: exzile --- src/inference_executor.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/inference_executor.hpp b/src/inference_executor.hpp index 1092c82f7c..9804f1c145 100644 --- a/src/inference_executor.hpp +++ b/src/inference_executor.hpp @@ -207,9 +207,17 @@ Status modelInferAsync(ModelInstance& instance, const RequestType* request, inferRequest.start_async(); } catch (std::exception& e) { SPDLOG_DEBUG("caught exception in ov::InferRequest.start_async: {}", e.what()); + // start_async threw, so the completion callback will never fire to release the + // captured stream-id guard; clear the callback to destroy it and return the + // infer-request slot to the pool (otherwise the slot leaks -> #2871). + inferRequest.set_callback([](std::exception_ptr) {}); return StatusCode::OV_INTERNAL_INFERENCE_ERROR; } catch (...) { SPDLOG_DEBUG("caught exception in ov::InferRequest.start_async"); + // start_async threw, so the completion callback will never fire to release the + // captured stream-id guard; clear the callback to destroy it and return the + // infer-request slot to the pool (otherwise the slot leaks -> #2871). + inferRequest.set_callback([](std::exception_ptr) {}); return StatusCode::OV_INTERNAL_INFERENCE_ERROR; } return StatusCode::OK; From 93e98f02229afff8defd9709a42d821de7849379 Mon Sep 17 00:00:00 2001 From: exzile Date: Sat, 27 Jun 2026 21:35:43 -0400 Subject: [PATCH 2/2] Fall back to legacy pipeline when CB model lacks PagedAttention Models whose IR exports without a ScaledDotProductAttention op (e.g. Gemma) cannot undergo the SDPAToPagedAttention transformation required by the continuous batching engine. Previously, when the pipeline type was auto-selected, OVMS picked *_CB and then failed hard during LLM node initialization, taking the whole server down with a cryptic SDPAToPagedAttention error (see model_server#4178). Detect this specific failure in the continuous batching initializer and report it via a dedicated status (LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED), logged at warning level. When the pipeline type was auto-selected, fall back to the legacy pipeline (LM_CB->LM, VLM_CB->VLM). An explicitly requested *_CB pipeline still surfaces the error rather than being silently downgraded. Verified manually with OpenVINO/gemma-4-E4B-it-int4-ov: AUTO now falls back to the legacy VLM servable and serves chat completions instead of failing to start. Co-Authored-By: Claude Opus 4.8 --- .../continuous_batching/servable_initializer.cpp | 11 ++++++++++- src/llm/servable_initializer.cpp | 15 +++++++++++++++ src/status.cpp | 1 + src/status.hpp | 1 + 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/llm/language_model/continuous_batching/servable_initializer.cpp b/src/llm/language_model/continuous_batching/servable_initializer.cpp index 1aaff99844..f3567b26ff 100644 --- a/src/llm/language_model/continuous_batching/servable_initializer.cpp +++ b/src/llm/language_model/continuous_batching/servable_initializer.cpp @@ -253,7 +253,16 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptrpluginConfig, properties->tokenizerPluginConfig); properties->tokenizer = properties->pipeline->get_tokenizer(); } catch (const std::exception& e) { - SPDLOG_ERROR("Error during llm node initialization for models_path: {} exception: {}", parsedModelsPath, e.what()); + // Some architectures (e.g. Gemma) export without a ScaledDotProductAttention op, so the + // SDPAToPagedAttention transformation required by continuous batching cannot be applied. + // Report this distinctly (at warning level) so callers can fall back to the legacy pipeline + // when appropriate, rather than logging it as a hard error on an otherwise-recoverable path. + const std::string error = e.what(); + if (error.find("ScaledDotProductAttention") != std::string::npos) { + SPDLOG_WARN("Model at models_path: {} does not support PagedAttention required by continuous batching: {}", parsedModelsPath, error); + return StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED; + } + SPDLOG_ERROR("Error during llm node initialization for models_path: {} exception: {}", parsedModelsPath, error); return StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED; } catch (...) { SPDLOG_ERROR("Error during llm node initialization for models_path: {}", parsedModelsPath); diff --git a/src/llm/servable_initializer.cpp b/src/llm/servable_initializer.cpp index 90673fdbc3..7e571848a8 100644 --- a/src/llm/servable_initializer.cpp +++ b/src/llm/servable_initializer.cpp @@ -430,11 +430,21 @@ Status initializeGenAiServable(std::shared_ptr& servable, const : if (status != StatusCode::OK) { return status; } + // When the pipeline type was auto-selected (not explicitly requested by the user), continuous + // batching can gracefully fall back to the legacy pipeline for models that do not support + // PagedAttention (e.g. Gemma, exported without a ScaledDotProductAttention op). When the user + // explicitly requested *_CB we surface the error instead of silently changing their request. + const bool autoSelectedPipelineType = (nodeOptions.pipeline_type() == mediapipe::LLMCalculatorOptions::AUTO); if (pipelineType == PipelineType::LM_CB) { SPDLOG_LOGGER_INFO(modelmanager_logger, "Initializing Language Model Continuous Batching servable"); ContinuousBatchingServableInitializer cbServableInitializer; servable = std::make_shared(); status = cbServableInitializer.initialize(servable, nodeOptions, graphPath); + if (status == StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED && autoSelectedPipelineType) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Model does not support PagedAttention, falling back to Language Model Legacy servable"); + LegacyServableInitializer legacyServableInitializer; + status = legacyServableInitializer.initialize(servable, nodeOptions, graphPath); + } if (status != StatusCode::OK) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during LLM node resources initialization: {}", status.string()); return status; @@ -446,6 +456,11 @@ Status initializeGenAiServable(std::shared_ptr& servable, const : ContinuousBatchingServableInitializer cbServableInitializer; servable = std::make_shared(); status = cbServableInitializer.initialize(servable, nodeOptions, graphPath); + if (status == StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED && autoSelectedPipelineType) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Model does not support PagedAttention, falling back to Visual Language Model Legacy servable"); + VisualLanguageModelLegacyServableInitializer legacyServableInitializer; + status = legacyServableInitializer.initialize(servable, nodeOptions, graphPath); + } if (status != StatusCode::OK) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during LLM node resources initialization: {}", status.string()); return status; diff --git a/src/status.cpp b/src/status.cpp index 0394192e86..2a73ff0024 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -226,6 +226,7 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::LLM_NODE_DIRECTORY_DOES_NOT_EXIST, "The LLM Node workspace path does not exist"}, {StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, "The LLM Node workspace path does not exist and not gguf model file"}, {StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, "The LLM Node resource initialization failed"}, + {StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED, "The model does not support PagedAttention required by the continuous batching pipeline"}, {StatusCode::LLM_NODE_MISSING_OPTIONS, "The LLM Node is missing options definition"}, {StatusCode::LLM_NODE_MISSING_NAME, "The LLM Node is missing name definition"}, diff --git a/src/status.hpp b/src/status.hpp index 94be7948cb..dbfc6763eb 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -272,6 +272,7 @@ enum class StatusCode { LLM_NODE_DIRECTORY_DOES_NOT_EXIST, LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, + LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED, LLM_NODE_MISSING_OPTIONS, LLM_NODE_MISSING_NAME,