From 6cbd37a8cc20ef2dab208b8b09f0b6fe9cb8aee2 Mon Sep 17 00:00:00 2001 From: Human <5217366+BrainSlugs83@users.noreply.github.com> Date: Thu, 4 Jun 2026 02:16:56 -0700 Subject: [PATCH] Preserve sideload tool call metadata Read optional toolCallStart, toolCallEnd, and supportsToolCalling from sideloaded inference_model.json files and thread the values through catalog BYO ModelInfo synthesis so local ONNX models can surface structured tool calls. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk_v2/cpp/src/catalog/azure_model_catalog.cc | 13 ++--- sdk_v2/cpp/src/catalog/catalog_client.cc | 32 ++++++++++- sdk_v2/cpp/src/catalog/catalog_client.h | 6 +++ sdk_v2/cpp/src/catalog/local_model_scanner.cc | 54 ++++++++++++++----- sdk_v2/cpp/src/catalog/local_model_scanner.h | 15 ++++++ .../internal_api/local_model_scanner_test.cc | 50 ++++++++++++++++- 6 files changed, 143 insertions(+), 27 deletions(-) diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 61da8ffd7..23a7d3013 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -69,28 +69,23 @@ std::vector AzureModelCatalog::FetchModels() const { "Getting latest info from the Azure catalog and for locally cached models."); // Discover locally cached models. - auto local_models = ScanLocalModels(cache_dir, logger_); - std::vector cached_model_ids; - cached_model_ids.reserve(local_models.size()); - for (const auto& [id, path] : local_models) { - cached_model_ids.push_back(id); - } + auto local_models = ScanLocalModelInfos(cache_dir, logger_); logger_.Log(LogLevel::Information, - fmt::format("Found {} locally cached models.", cached_model_ids.size())); + fmt::format("Found {} locally cached models.", local_models.size())); auto fetch_from = [&](const std::string& url, const std::optional& filter) { // Preserve byte-identical behavior for the "no override" case (previously stored as ""), // while letting callers explicitly request "" as a real filter override. auto client = MakeCatalogClient(url, filter.value_or(""), ep_detector_, logger_, cache_dir); - auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_); + auto model_infos = FetchAllModelInfosWithCachedModels(*client, local_models, logger_); for (const auto& info : model_infos) { // Check if the model is locally cached and pass the path if so. std::string local_path; auto it = local_models.find(info.model_id); if (it != local_models.end()) { - local_path = it->second; + local_path = it->second.path; } models.push_back(model_factory_(ModelInfo(info), std::move(local_path))); diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index 31ae5c3ab..692008522 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -47,10 +47,22 @@ std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, ILogger& logger) { + std::map cached_models; + for (const auto& id : cached_model_ids) { + cached_models[id] = {}; + } + + return FetchAllModelInfosWithCachedModels(client, cached_models, logger); +} + +std::vector FetchAllModelInfosWithCachedModels( + ICatalogClient& client, + const std::map& cached_models, + ILogger& logger) { // Step 1: Fetch latest catalog models (existing flow). auto result = client.FetchAllModelInfos(); - if (cached_model_ids.empty()) { + if (cached_models.empty()) { return result; } @@ -61,7 +73,8 @@ std::vector FetchAllModelInfosWithCachedModels( } std::vector unresolved_ids; - for (const auto& id : cached_model_ids) { + for (const auto& cached_model : cached_models) { + const auto& id = cached_model.first; if (resolved_ids.find(id) == resolved_ids.end()) { unresolved_ids.push_back(id); } @@ -98,6 +111,21 @@ std::vector FetchAllModelInfosWithCachedModels( info.version = version; info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR] = "Local"; info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX"; + const auto metadata = cached_models.find(id); + if (metadata != cached_models.end()) { + if (!metadata->second.tool_call_start.empty()) { + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR] = metadata->second.tool_call_start; + } + + if (!metadata->second.tool_call_end.empty()) { + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR] = metadata->second.tool_call_end; + } + + if (metadata->second.supports_tool_calling.has_value()) { + info.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT] = + *metadata->second.supports_tool_calling; + } + } result.push_back(std::move(info)); } diff --git a/sdk_v2/cpp/src/catalog/catalog_client.h b/sdk_v2/cpp/src/catalog/catalog_client.h index d560cbe59..1de71c3c6 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.h +++ b/sdk_v2/cpp/src/catalog/catalog_client.h @@ -3,6 +3,7 @@ #pragma once #include "ep_detection/ep_detector.h" +#include "catalog/local_model_scanner.h" #include "logger.h" #include "model_info.h" @@ -36,6 +37,11 @@ std::vector FetchAllModelInfosWithCachedModels( const std::vector& cached_model_ids, ILogger& logger); +std::vector FetchAllModelInfosWithCachedModels( + ICatalogClient& client, + const std::map& cached_models, + ILogger& logger); + /// Construct a catalog client. Dispatches based on `base_url`: /// - "static" -> returns a client backed by the embedded snapshot. Ignores /// `filter_override` and `cache_directory`. Filters models by the diff --git a/sdk_v2/cpp/src/catalog/local_model_scanner.cc b/sdk_v2/cpp/src/catalog/local_model_scanner.cc index 3d41661d5..4402b74d8 100644 --- a/sdk_v2/cpp/src/catalog/local_model_scanner.cc +++ b/sdk_v2/cpp/src/catalog/local_model_scanner.cc @@ -19,29 +19,43 @@ const char* kGenAIConfigFileName = "genai_config.json"; const char* kDownloadSignalFileName = "download.tmp"; const char* kInferenceModelFileName = "inference_model.json"; -/// Try to read the model name from inference_model.json in the given directory. -/// Returns the model name string, or empty string on failure. -std::string ReadModelNameFromInferenceModel(const fs::path& dir) { +/// Try to read model metadata from inference_model.json in the given directory. +/// Returns false on failure. +bool ReadModelInfoFromInferenceModel(const fs::path& dir, + std::string& model_name, + LocalModelScanResult& info) { auto path = dir / kInferenceModelFileName; if (!fs::exists(path)) { - return {}; + return false; } try { std::ifstream file(path); if (!file.is_open()) { - return {}; + return false; } auto j = nlohmann::json::parse(file); if (j.contains("Name") && j["Name"].is_string()) { - return j["Name"].get(); + model_name = j["Name"].get(); + } + + if (j.contains("toolCallStart") && j["toolCallStart"].is_string()) { + info.tool_call_start = j["toolCallStart"].get(); + } + + if (j.contains("toolCallEnd") && j["toolCallEnd"].is_string()) { + info.tool_call_end = j["toolCallEnd"].get(); + } + + if (j.contains("supportsToolCalling") && j["supportsToolCalling"].is_boolean()) { + info.supports_tool_calling = j["supportsToolCalling"].get() ? 1 : 0; } } catch (...) { // Ignore parse errors for individual files. } - return {}; + return !model_name.empty(); } /// Check if a directory is a valid cached model directory. @@ -64,7 +78,7 @@ bool IsValidModelDirectory(const fs::path& dir) { /// Recursively scan a directory for valid model directories. void ScanDirectory(const fs::path& dir, - std::map& results, + std::map& results, ILogger& logger) { try { if (!fs::exists(dir) || !fs::is_directory(dir)) { @@ -73,14 +87,16 @@ void ScanDirectory(const fs::path& dir, // Check if this directory itself is a valid model directory. if (IsValidModelDirectory(dir)) { - auto model_name = ReadModelNameFromInferenceModel(dir); - if (!model_name.empty()) { + std::string model_name; + LocalModelScanResult info; + if (ReadModelInfoFromInferenceModel(dir, model_name, info)) { // If the model name doesn't contain a ':' version separator, append ":0". if (model_name.find(':') == std::string::npos) { model_name += ":0"; } - results[model_name] = dir.string(); + info.path = dir.string(); + results[model_name] = std::move(info); } // Don't recurse into a valid model directory — it's a leaf. @@ -104,9 +120,9 @@ void ScanDirectory(const fs::path& dir, } // anonymous namespace -std::map ScanLocalModels(const std::string& cache_directory, - ILogger& logger) { - std::map results; +std::map ScanLocalModelInfos(const std::string& cache_directory, + ILogger& logger) { + std::map results; if (cache_directory.empty()) { return results; @@ -134,4 +150,14 @@ std::map ScanLocalModels(const std::string& cache_dire return results; } +std::map ScanLocalModels(const std::string& cache_directory, + ILogger& logger) { + std::map results; + for (const auto& [id, info] : ScanLocalModelInfos(cache_directory, logger)) { + results[id] = info.path; + } + + return results; +} + } // namespace fl diff --git a/sdk_v2/cpp/src/catalog/local_model_scanner.h b/sdk_v2/cpp/src/catalog/local_model_scanner.h index 0add84aa6..20f1cfd52 100644 --- a/sdk_v2/cpp/src/catalog/local_model_scanner.h +++ b/sdk_v2/cpp/src/catalog/local_model_scanner.h @@ -5,10 +5,25 @@ #include "logger.h" #include +#include #include namespace fl { +struct LocalModelScanResult { + std::string path; + std::string tool_call_start; + std::string tool_call_end; + std::optional supports_tool_calling; +}; + +/// Scan a model cache directory for locally cached (downloaded) models. +/// Returns a map of model_id -> scan metadata for each valid model found. +/// A valid model directory has genai_config.json, no download.tmp, +/// and an inference_model.json with a model name. +std::map ScanLocalModelInfos(const std::string& cache_directory, + ILogger& logger); + /// Scan a model cache directory for locally cached (downloaded) models. /// Returns a map of model_id -> local_path for each valid model found. /// A valid model directory has genai_config.json, no download.tmp, diff --git a/sdk_v2/cpp/test/internal_api/local_model_scanner_test.cc b/sdk_v2/cpp/test/internal_api/local_model_scanner_test.cc index 9c577258c..0110b0a68 100644 --- a/sdk_v2/cpp/test/internal_api/local_model_scanner_test.cc +++ b/sdk_v2/cpp/test/internal_api/local_model_scanner_test.cc @@ -3,7 +3,9 @@ // // Tests for ScanLocalModels — discovering locally cached model directories. // +#include "catalog/catalog_client.h" #include "catalog/local_model_scanner.h" +#include #include "logger.h" #include @@ -11,6 +13,7 @@ #include #include #include +#include #ifdef _WIN32 #include @@ -21,6 +24,19 @@ using namespace fl; namespace fs = std::filesystem; +namespace { + +class EmptyCatalogClient : public ICatalogClient { + public: + std::vector FetchAllModelInfos() override { return {}; } + + std::vector FetchModelsByIds(const std::vector&) override { + return {}; + } +}; + +} // namespace + // ======================================================================== // Test fixture — creates a unique temp directory per test // ======================================================================== @@ -39,7 +55,8 @@ class LocalModelScannerTest : public ::testing::Test { /// Create a valid model directory with genai_config.json and inference_model.json. void CreateModelDir(const std::string& relative_path, - const std::string& model_name) { + const std::string& model_name, + const std::string& extra_json = "") { auto dir = fs::path(test_dir_) / relative_path; fs::create_directories(dir); @@ -52,7 +69,7 @@ class LocalModelScannerTest : public ::testing::Test { // inference_model.json — Name field is read { std::ofstream f(dir / "inference_model.json"); - f << R"({"Name": ")" << model_name << R"(", "PromptTemplate": null})"; + f << R"({"Name": ")" << model_name << R"(", "PromptTemplate": null)" << extra_json << "}"; } } @@ -91,6 +108,35 @@ TEST_F(LocalModelScannerTest, ValidModelDirectory) { EXPECT_EQ(results["phi-4-mini:3"], expected_path); } +TEST_F(LocalModelScannerTest, SideloadToolCallMarkersFlowIntoSynthesizedBYOModelInfo) { + CreateModelDir("publisher/x", "x:1", + R"(, "toolCallStart": "", "toolCallEnd": "", "supportsToolCalling": true)"); + + auto scanned = ScanLocalModelInfos(test_dir_, logger_); + EmptyCatalogClient client; + auto results = FetchAllModelInfosWithCachedModels(client, scanned, logger_); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].model_id, "x:1"); + EXPECT_EQ(results[0].string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR), ""); + EXPECT_EQ(results[0].string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR), ""); + EXPECT_EQ(results[0].int_properties.at(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT), 1); +} + +TEST_F(LocalModelScannerTest, SideloadWithoutToolCallMarkersPreservesAbsentProperties) { + CreateModelDir("publisher/y", "y:1"); + + auto scanned = ScanLocalModelInfos(test_dir_, logger_); + EmptyCatalogClient client; + auto results = FetchAllModelInfosWithCachedModels(client, scanned, logger_); + + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results[0].model_id, "y:1"); + EXPECT_EQ(results[0].string_properties.count(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR), 0u); + EXPECT_EQ(results[0].string_properties.count(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR), 0u); + EXPECT_EQ(results[0].int_properties.count(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT), 0u); +} + TEST_F(LocalModelScannerTest, VersionlessModelNameGetsZeroAppended) { CreateModelDir("publisher/my-model", "my-model");