diff --git a/src/inference_executor.hpp b/src/inference_executor.hpp index 1092c82f7c..9804f1c145 100644 --- a/src/inference_executor.hpp +++ b/src/inference_executor.hpp @@ -207,9 +207,17 @@ Status modelInferAsync(ModelInstance& instance, const RequestType* request, inferRequest.start_async(); } catch (std::exception& e) { SPDLOG_DEBUG("caught exception in ov::InferRequest.start_async: {}", e.what()); + // start_async threw, so the completion callback will never fire to release the + // captured stream-id guard; clear the callback to destroy it and return the + // infer-request slot to the pool (otherwise the slot leaks -> #2871). + inferRequest.set_callback([](std::exception_ptr) {}); return StatusCode::OV_INTERNAL_INFERENCE_ERROR; } catch (...) { SPDLOG_DEBUG("caught exception in ov::InferRequest.start_async"); + // start_async threw, so the completion callback will never fire to release the + // captured stream-id guard; clear the callback to destroy it and return the + // infer-request slot to the pool (otherwise the slot leaks -> #2871). + inferRequest.set_callback([](std::exception_ptr) {}); return StatusCode::OV_INTERNAL_INFERENCE_ERROR; } return StatusCode::OK; diff --git a/src/llm/language_model/continuous_batching/servable_initializer.cpp b/src/llm/language_model/continuous_batching/servable_initializer.cpp index 1aaff99844..f3567b26ff 100644 --- a/src/llm/language_model/continuous_batching/servable_initializer.cpp +++ b/src/llm/language_model/continuous_batching/servable_initializer.cpp @@ -253,7 +253,16 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptrpluginConfig, properties->tokenizerPluginConfig); properties->tokenizer = properties->pipeline->get_tokenizer(); } catch (const std::exception& e) { - SPDLOG_ERROR("Error during llm node initialization for models_path: {} exception: {}", parsedModelsPath, e.what()); + // Some architectures (e.g. Gemma) export without a ScaledDotProductAttention op, so the + // SDPAToPagedAttention transformation required by continuous batching cannot be applied. + // Report this distinctly (at warning level) so callers can fall back to the legacy pipeline + // when appropriate, rather than logging it as a hard error on an otherwise-recoverable path. + const std::string error = e.what(); + if (error.find("ScaledDotProductAttention") != std::string::npos) { + SPDLOG_WARN("Model at models_path: {} does not support PagedAttention required by continuous batching: {}", parsedModelsPath, error); + return StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED; + } + SPDLOG_ERROR("Error during llm node initialization for models_path: {} exception: {}", parsedModelsPath, error); return StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED; } catch (...) { SPDLOG_ERROR("Error during llm node initialization for models_path: {}", parsedModelsPath); diff --git a/src/llm/servable_initializer.cpp b/src/llm/servable_initializer.cpp index 1d09334620..027255cea7 100644 --- a/src/llm/servable_initializer.cpp +++ b/src/llm/servable_initializer.cpp @@ -437,11 +437,21 @@ Status initializeGenAiServable(std::shared_ptr& servable, const : if (status != StatusCode::OK) { return status; } + // When the pipeline type was auto-selected (not explicitly requested by the user), continuous + // batching can gracefully fall back to the legacy pipeline for models that do not support + // PagedAttention (e.g. Gemma, exported without a ScaledDotProductAttention op). When the user + // explicitly requested *_CB we surface the error instead of silently changing their request. + const bool autoSelectedPipelineType = (nodeOptions.pipeline_type() == mediapipe::LLMCalculatorOptions::AUTO); if (pipelineType == PipelineType::LM_CB) { SPDLOG_LOGGER_INFO(modelmanager_logger, "Initializing Language Model Continuous Batching servable"); ContinuousBatchingServableInitializer cbServableInitializer; servable = std::make_shared(); status = cbServableInitializer.initialize(servable, nodeOptions, graphPath); + if (status == StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED && autoSelectedPipelineType) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Model does not support PagedAttention, falling back to Language Model Legacy servable"); + LegacyServableInitializer legacyServableInitializer; + status = legacyServableInitializer.initialize(servable, nodeOptions, graphPath); + } if (status != StatusCode::OK) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during LLM node resources initialization: {}", status.string()); return status; @@ -453,6 +463,11 @@ Status initializeGenAiServable(std::shared_ptr& servable, const : ContinuousBatchingServableInitializer cbServableInitializer; servable = std::make_shared(); status = cbServableInitializer.initialize(servable, nodeOptions, graphPath); + if (status == StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED && autoSelectedPipelineType) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Model does not support PagedAttention, falling back to Visual Language Model Legacy servable"); + VisualLanguageModelLegacyServableInitializer legacyServableInitializer; + status = legacyServableInitializer.initialize(servable, nodeOptions, graphPath); + } if (status != StatusCode::OK) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during LLM node resources initialization: {}", status.string()); return status; diff --git a/src/status.cpp b/src/status.cpp index 0394192e86..2a73ff0024 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -226,6 +226,7 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::LLM_NODE_DIRECTORY_DOES_NOT_EXIST, "The LLM Node workspace path does not exist"}, {StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, "The LLM Node workspace path does not exist and not gguf model file"}, {StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, "The LLM Node resource initialization failed"}, + {StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED, "The model does not support PagedAttention required by the continuous batching pipeline"}, {StatusCode::LLM_NODE_MISSING_OPTIONS, "The LLM Node is missing options definition"}, {StatusCode::LLM_NODE_MISSING_NAME, "The LLM Node is missing name definition"}, diff --git a/src/status.hpp b/src/status.hpp index 94be7948cb..dbfc6763eb 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -272,6 +272,7 @@ enum class StatusCode { LLM_NODE_DIRECTORY_DOES_NOT_EXIST, LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, + LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED, LLM_NODE_MISSING_OPTIONS, LLM_NODE_MISSING_NAME,