From 64865b583c6f9eb0d32ef6531efae57d9458f249 Mon Sep 17 00:00:00 2001 From: denispetre Date: Thu, 27 Aug 2026 11:39:51 +0300 Subject: [PATCH 1/2] ci: dispatchable model-onboarding check for a single model as a given product Runs the langchain integration suite for one model that is not in the committed matrix (injected for the run only) as agents/agenthub/agent-gym/testproduct, against the LLM Gateway env held in a GitHub environment (LLMGW_SETTINGS = alpha; a prod-tier environment can be selected once provisioned). Dispatched by hand or by UiPath/llm-gateway-config when a model is routed for agents/agenthub in prod. Co-Authored-By: Claude Fable 5 --- .github/workflows/model-onboarding.yml | 106 +++++++++++++++++++++++++ scripts/inject_model_into_matrix.py | 67 ++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 .github/workflows/model-onboarding.yml create mode 100644 scripts/inject_model_into_matrix.py diff --git a/.github/workflows/model-onboarding.yml b/.github/workflows/model-onboarding.yml new file mode 100644 index 0000000..3b71f5b --- /dev/null +++ b/.github/workflows/model-onboarding.yml @@ -0,0 +1,106 @@ +# Model onboarding check: run the langchain integration suite for ONE model that is not (yet) in +# the committed test matrix — e.g. a model the LLM Gateway just onboarded — as a given requesting +# product, against the LLM Gateway environment held in a GitHub environment of this repo. +# +# Dispatched by hand, or by UiPath/llm-gateway-config when a model is routed for agents/agenthub +# in prod. The model is injected into the per-provider conftest matrix for this run only. +# +# target_environment: LLMGW_SETTINGS (alpha, exists) or a prod-tier environment such as +# LLMGW_SETTINGS_PROD — same variable/secret names as ci.yml's test job. +name: Model onboarding check + +on: + workflow_dispatch: + inputs: + model: + description: model id as passed to the gateway (e.g. gemini-3.7-flash) + required: true + type: string + family: + description: LLM Gateway ModelFamily + required: true + type: choice + options: [GoogleGemini, AnthropicClaude, OpenAi] + vendor: + description: LLM Gateway Vendor + required: true + type: choice + options: [VertexAi, AwsBedrock, OpenAi, NativeOpenAi] + product: + description: X-UiPath-LlmGateway-RequestingProduct + required: true + type: choice + options: [agents, agenthub, agent-gym, testproduct] + target_environment: + description: GitHub environment holding the LLMGW_* settings + required: false + default: LLMGW_SETTINGS + type: string + +permissions: + contents: read + +jobs: + test: + name: ${{ inputs.model }} · ${{ inputs.product }} · ${{ inputs.target_environment }} + runs-on: uipath-ubuntu-latest + environment: ${{ inputs.target_environment }} + env: + UIPATH_LLM_SERVICE: ${{ vars.UIPATH_LLM_SERVICE }} + LLMGW_URL: ${{ vars.LLMGW_URL }} + LLMGW_SEMANTIC_ORG_ID: ${{ secrets.LLMGW_SEMANTIC_ORG_ID }} + LLMGW_SEMANTIC_USER_ID: ${{ secrets.LLMGW_SEMANTIC_USER_ID }} + LLMGW_SEMANTIC_TENANT_ID: ${{ secrets.LLMGW_SEMANTIC_TENANT_ID }} + LLMGW_CLIENT_ID: ${{ secrets.LLMGW_CLIENT_ID }} + LLMGW_CLIENT_SECRET: ${{ secrets.LLMGW_CLIENT_SECRET }} + LLMGW_REQUESTING_PRODUCT: ${{ inputs.product }} + LLMGW_REQUESTING_FEATURE: ${{ inputs.product == 'agenthub' && 'llm-call' || inputs.product == 'agents' && 'agents-prompt' || vars.LLMGW_REQUESTING_FEATURE }} + LLMGW_OPERATION_CODE: ${{ inputs.product == 'agenthub' && 'AgentHub.LLM' || inputs.product == 'agents' && 'Agents.Execution' || '' }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + lfs: true + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + version: "0.9.27" + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version-file: ".python-version" + + - name: Install dependencies + run: uv sync --dev --all-extras + + - name: Inject the model into the test matrix (this run only) + id: inject + run: | + out=$(python scripts/inject_model_into_matrix.py --repo . \ + --model "${{ inputs.model }}" --family "${{ inputs.family }}" --vendor "${{ inputs.vendor }}") + echo "$out" + echo "dirs=$(echo "$out" | tail -1)" >> "$GITHUB_OUTPUT" + + # record_mode=new_episodes: an unseen model goes to the wire. The cassette DB is not + # uploaded or committed, so this never pollutes the committed recordings. + - name: Run the langchain integration suite for the model + run: | + paths=""; for d in ${{ steps.inject.outputs.dirs }}; do paths="$paths tests/langchain/clients/$d/test_integration.py"; done + uv run pytest $paths -k "${{ inputs.model }}" -q --no-header \ + --junitxml "report-${{ inputs.model }}-${{ inputs.product }}.xml" | tee "run-${{ inputs.model }}-${{ inputs.product }}.log" + + - name: Summary + if: always() + run: | + echo "### ${{ inputs.model }} · ${{ inputs.product }} · ${{ inputs.target_environment }}" >> "$GITHUB_STEP_SUMMARY" + tail -3 "run-${{ inputs.model }}-${{ inputs.product }}.log" >> "$GITHUB_STEP_SUMMARY" || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: model-onboarding-${{ inputs.model }}-${{ inputs.product }}-${{ inputs.target_environment }} + path: | + report-*.xml + run-*.log diff --git a/scripts/inject_model_into_matrix.py b/scripts/inject_model_into_matrix.py new file mode 100644 index 0000000..4ab9d39 --- /dev/null +++ b/scripts/inject_model_into_matrix.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Add a model to uipath-llm-client-python's langchain integration matrix (workflow-local edit, see .github/workflows/model-onboarding.yml). + +The client repo parameterizes its integration tests from per-provider conftest dicts +(COMPLETIONS_MODELS_WITH_CONFIGS). This inserts the model into the normalized-client dict +(UiPathChat, cross-provider) and the matching vendor-client dict, picking the thinking/ +reasoning config list by model family the same way the existing entries do. + + inject_model_into_matrix.py --repo . --model --family --vendor +""" +import argparse +import re +import sys +from pathlib import Path + +CLIENTS = Path("tests/langchain/clients") + + +def pick(model: str, family: str, vendor: str) -> list[tuple[str, str]]: + """(conftest dir, CONFIGS list name) pairs to insert into.""" + m = model.lower() + if family == "GoogleGemini": + gem = "GEMINI_3_CONFIGS" if re.search(r"gemini-3", m) else "GEMINI_2_5_CONFIGS" + return [("normalized", gem), ("google", gem)] + if family == "AnthropicClaude": + if vendor == "AwsBedrock" or m.startswith("anthropic."): + return [("normalized", "CLAUDE_BEDROCK_CONFIGS"), ("bedrock", "CLAUDE_BEDROCK_CONFIGS")] + return [("normalized", "CLAUDE_VERTEXAI_CONFIGS"), ("vertexai", "CLAUDE_VERTEXAI_CONFIGS")] + if family in ("OpenAi", "OpenAI", "Gpt", "GPT"): + reasoning = bool(re.match(r"(gpt-5|o\d)", m)) and "chat" not in m + return [("normalized", "GPT_REASONING_CONFIGS" if reasoning else "GPT_NON_REASONING_CONFIGS"), + ("openai", "GPT_MODELS_WITH_REASONING_CONFIGS" if reasoning else "GPT_MODELS_NON_REASONING_CONFIGS")] + return [("normalized", "GPT_NON_REASONING_CONFIGS")] + + +def inject(conftest: Path, model: str, configs: str) -> bool: + text = conftest.read_text(encoding="utf-8") + if f'"{model}"' in text: + return False + if configs not in text: + raise SystemExit(f"{conftest}: config list {configs} not found") + new, n = re.subn(r"(COMPLETIONS_MODELS_WITH_CONFIGS = \{\n)", rf'\1 "{model}": {configs},\n', text, count=1) + if n != 1: + raise SystemExit(f"{conftest}: COMPLETIONS_MODELS_WITH_CONFIGS dict not found") + conftest.write_text(new, encoding="utf-8") + return True + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--repo", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--family", required=True) + ap.add_argument("--vendor", required=True) + a = ap.parse_args() + touched = [] + for d, configs in pick(a.model, a.family, a.vendor): + f = Path(a.repo) / CLIENTS / d / "conftest.py" + if inject(f, a.model, configs): + touched.append(f"{d}:{configs}") + print(f"injected {a.model} into: {' '.join(touched) or '(already present)'}") + print(" ".join(d for d, _ in pick(a.model, a.family, a.vendor))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e5c60734fd64c727a0384b7735f6f3961fc25a96 Mon Sep 17 00:00:00 2001 From: denispetre Date: Thu, 27 Aug 2026 11:48:36 +0300 Subject: [PATCH 2/2] ci(model-onboarding): select vendor clients by the gateway's SupportedApiFlavors Each vendor client speaks one gateway API flavor; run a model only through the clients whose flavor it declares (Gemini generateContent, Anthropic Messages, Bedrock Invoke vs Converse, OpenAI chat-completions vs responses), composing the bedrock/openai/litellm entries per flavor instead of guessing by family. New api_flavors + operations inputs. One documented exception: Vertex-hosted Claude entries declare GeminiGenerateContent as a stand-in for Vertex passthrough, which for Claude is Anthropic Messages. Co-Authored-By: Claude Fable 5 --- .github/workflows/model-onboarding.yml | 15 ++- scripts/inject_model_into_matrix.py | 139 +++++++++++++++++++------ 2 files changed, 119 insertions(+), 35 deletions(-) diff --git a/.github/workflows/model-onboarding.yml b/.github/workflows/model-onboarding.yml index 3b71f5b..f4826b5 100644 --- a/.github/workflows/model-onboarding.yml +++ b/.github/workflows/model-onboarding.yml @@ -3,7 +3,8 @@ # product, against the LLM Gateway environment held in a GitHub environment of this repo. # # Dispatched by hand, or by UiPath/llm-gateway-config when a model is routed for agents/agenthub -# in prod. The model is injected into the per-provider conftest matrix for this run only. +# in prod. The model is injected into the per-provider conftest matrix for this run only, and only +# into the vendor clients whose API flavor the gateway declares for it (SupportedApiFlavors). # # target_environment: LLMGW_SETTINGS (alpha, exists) or a prod-tier environment such as # LLMGW_SETTINGS_PROD — same variable/secret names as ci.yml's test job. @@ -31,6 +32,15 @@ on: required: true type: choice options: [agents, agenthub, agent-gym, testproduct] + api_flavors: + description: comma-separated LLM Gateway SupportedApiFlavors (selects which vendor clients run) + required: true + type: string + operations: + description: comma-separated SupportedNormalizedOperations + required: false + default: NormalizedChat + type: string target_environment: description: GitHub environment holding the LLMGW_* settings required: false @@ -79,7 +89,8 @@ jobs: id: inject run: | out=$(python scripts/inject_model_into_matrix.py --repo . \ - --model "${{ inputs.model }}" --family "${{ inputs.family }}" --vendor "${{ inputs.vendor }}") + --model "${{ inputs.model }}" --family "${{ inputs.family }}" --vendor "${{ inputs.vendor }}" \ + --flavors "${{ inputs.api_flavors }}" --operations "${{ inputs.operations }}") echo "$out" echo "dirs=$(echo "$out" | tail -1)" >> "$GITHUB_OUTPUT" diff --git a/scripts/inject_model_into_matrix.py b/scripts/inject_model_into_matrix.py index 4ab9d39..fa8b5a1 100644 --- a/scripts/inject_model_into_matrix.py +++ b/scripts/inject_model_into_matrix.py @@ -1,12 +1,24 @@ #!/usr/bin/env python3 -"""Add a model to uipath-llm-client-python's langchain integration matrix (workflow-local edit, see .github/workflows/model-onboarding.yml). +"""Add a model to the langchain integration matrix for ONE run, selecting clients by the +LLM Gateway's declared SupportedApiFlavors (see .github/workflows/model-onboarding.yml). -The client repo parameterizes its integration tests from per-provider conftest dicts -(COMPLETIONS_MODELS_WITH_CONFIGS). This inserts the model into the normalized-client dict -(UiPathChat, cross-provider) and the matching vendor-client dict, picking the thinking/ -reasoning config list by model family the same way the existing entries do. +Each vendor client speaks one gateway API flavor; a model is only exercised through clients +whose flavor it declares: - inject_model_into_matrix.py --repo . --model --family --vendor + GeminiGenerateContent -> google (UiPathChatGoogleGenerativeAI), litellm GEMINI_CONFIGS + AnthropicMessages -> anthropic (UiPathChatAnthropic, vendor_type by vendor) + + vertexai (UiPathChatAnthropicVertex) [vendor VertexAi] + + bedrock (UiPathChatAnthropicBedrock) [vendor AwsBedrock] + + litellm VERTEX_CLAUDE_CONFIGS [vendor VertexAi] + AwsBedrockInvoke -> bedrock (UiPathChatBedrock), litellm BEDROCK_INVOKE_CONFIGS + AwsBedrockConverse -> bedrock (UiPathChatBedrockConverse), litellm BEDROCK_CONVERSE_CONFIGS + OpenAiChatCompletions -> openai (UiPathAzureChatOpenAI), litellm OPENAI_CONFIGS + OpenAiResponses -> openai (UiPathAzureChatOpenAI, use_responses_api), litellm OPENAI_RESPONSES_CONFIGS + NormalizedChat (op) -> normalized (UiPathChat) always, thinking/reasoning config by family + + inject_model_into_matrix.py --repo . --model --family --vendor \ + --flavors GeminiGenerateContent[,...] [--operations NormalizedChat] +Prints the conftest dirs touched on the last stdout line (consumed by the workflow). """ import argparse import re @@ -14,32 +26,88 @@ from pathlib import Path CLIENTS = Path("tests/langchain/clients") +THINKING = '{"max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}}' + + +def entries(cls: str, extra: str | None = None) -> str: + """Config entries (plain + optional kwargs variant) as Python source, existing-list style.""" + base = f'{{"model_class": {cls}}}' + if extra is None: + return base + return f'{base}, {{"model_class": {cls}, "model_kwargs": {extra}}}' + + +def gemini_list(model: str) -> str: + return "GEMINI_3_CONFIGS" if re.search(r"gemini-3", model.lower()) else "GEMINI_2_5_CONFIGS" -def pick(model: str, family: str, vendor: str) -> list[tuple[str, str]]: - """(conftest dir, CONFIGS list name) pairs to insert into.""" +def gpt_reasoning(model: str) -> bool: m = model.lower() - if family == "GoogleGemini": - gem = "GEMINI_3_CONFIGS" if re.search(r"gemini-3", m) else "GEMINI_2_5_CONFIGS" - return [("normalized", gem), ("google", gem)] - if family == "AnthropicClaude": - if vendor == "AwsBedrock" or m.startswith("anthropic."): - return [("normalized", "CLAUDE_BEDROCK_CONFIGS"), ("bedrock", "CLAUDE_BEDROCK_CONFIGS")] - return [("normalized", "CLAUDE_VERTEXAI_CONFIGS"), ("vertexai", "CLAUDE_VERTEXAI_CONFIGS")] - if family in ("OpenAi", "OpenAI", "Gpt", "GPT"): - reasoning = bool(re.match(r"(gpt-5|o\d)", m)) and "chat" not in m - return [("normalized", "GPT_REASONING_CONFIGS" if reasoning else "GPT_NON_REASONING_CONFIGS"), - ("openai", "GPT_MODELS_WITH_REASONING_CONFIGS" if reasoning else "GPT_MODELS_NON_REASONING_CONFIGS")] - return [("normalized", "GPT_NON_REASONING_CONFIGS")] - - -def inject(conftest: Path, model: str, configs: str) -> bool: + return bool(re.match(r"(gpt-5|o\d)", m)) and "chat" not in m + + +def plan(model: str, family: str, vendor: str, flavors: set[str], operations: set[str]) -> dict[str, str]: + """conftest dir -> Python expression for the model's config list.""" + out: dict[str, str] = {} + lite: list[str] = [] + if "NormalizedChat" in operations: + if family == "GoogleGemini": + out["normalized"] = gemini_list(model) + elif family == "AnthropicClaude": + out["normalized"] = "CLAUDE_BEDROCK_CONFIGS" if vendor == "AwsBedrock" else "CLAUDE_VERTEXAI_CONFIGS" + else: + out["normalized"] = "GPT_REASONING_CONFIGS" if gpt_reasoning(model) else "GPT_NON_REASONING_CONFIGS" + # Gateway convention: Vertex-hosted Claude entries declare GeminiGenerateContent (the enum has no + # Vertex-Anthropic flavor); it means "Vertex passthrough", which for Claude is Anthropic Messages. + if family == "AnthropicClaude" and vendor == "VertexAi" and "GeminiGenerateContent" in flavors: + flavors = (flavors - {"GeminiGenerateContent"}) | {"AnthropicMessages"} + if "GeminiGenerateContent" in flavors: + out["google"] = gemini_list(model) + lite.append("GEMINI_CONFIGS") + if "AnthropicMessages" in flavors: + if vendor == "AwsBedrock": + out["anthropic"] = "CLAUDE_BEDROCK_CONFIGS" + else: + out["anthropic"] = "CLAUDE_VERTEXAI_CONFIGS" + out["vertexai"] = "CLAUDE_VERTEXAI_CONFIGS" + lite.append("VERTEX_CLAUDE_CONFIGS") + bedrock: list[str] = [] + if "AnthropicMessages" in flavors and vendor == "AwsBedrock": + bedrock.append(entries("UiPathChatAnthropicBedrock", THINKING)) + if "AwsBedrockInvoke" in flavors: + bedrock.append(entries("UiPathChatBedrock", THINKING)) + lite.append("BEDROCK_INVOKE_CONFIGS") + if "AwsBedrockConverse" in flavors: + bedrock.append(entries("UiPathChatBedrockConverse", THINKING)) + lite.append("BEDROCK_CONVERSE_CONFIGS") + if bedrock: + out["bedrock"] = "[" + ", ".join(bedrock) + "]" + openai: list[str] = [] + if "OpenAiChatCompletions" in flavors: + openai.append(entries("UiPathAzureChatOpenAI", '{"reasoning_effort": "low"}' if gpt_reasoning(model) else None)) + lite.append("OPENAI_CONFIGS") + if "OpenAiResponses" in flavors: + if gpt_reasoning(model): + kw = '{"use_responses_api": True, "reasoning": {"effort": "low", "summary": "auto"}, "verbosity": "low"}' + else: + kw = '{"use_responses_api": True}' + openai.append(f'{{"model_class": UiPathAzureChatOpenAI, "model_kwargs": {kw}}}') + lite.append("OPENAI_RESPONSES_CONFIGS") + if openai: + out["openai"] = "[" + ", ".join(openai) + "]" + if lite: + out["litellm"] = " + ".join(lite) + return out + + +def inject(conftest: Path, model: str, expr: str) -> bool: text = conftest.read_text(encoding="utf-8") if f'"{model}"' in text: return False - if configs not in text: - raise SystemExit(f"{conftest}: config list {configs} not found") - new, n = re.subn(r"(COMPLETIONS_MODELS_WITH_CONFIGS = \{\n)", rf'\1 "{model}": {configs},\n', text, count=1) + for name in re.findall(r"\b([A-Z][A-Z0-9_]+_CONFIGS)\b", expr): + if not re.search(rf"^{name} = \[", text, re.M): + raise SystemExit(f"{conftest}: config list {name} not found") + new, n = re.subn(r"(COMPLETIONS_MODELS_WITH_CONFIGS = \{\n)", rf'\1 "{model}": {expr},\n', text, count=1) if n != 1: raise SystemExit(f"{conftest}: COMPLETIONS_MODELS_WITH_CONFIGS dict not found") conftest.write_text(new, encoding="utf-8") @@ -52,14 +120,19 @@ def main() -> int: ap.add_argument("--model", required=True) ap.add_argument("--family", required=True) ap.add_argument("--vendor", required=True) + ap.add_argument("--flavors", required=True, help="comma-separated SupportedApiFlavors") + ap.add_argument("--operations", default="NormalizedChat", help="comma-separated SupportedNormalizedOperations") a = ap.parse_args() - touched = [] - for d, configs in pick(a.model, a.family, a.vendor): - f = Path(a.repo) / CLIENTS / d / "conftest.py" - if inject(f, a.model, configs): - touched.append(f"{d}:{configs}") - print(f"injected {a.model} into: {' '.join(touched) or '(already present)'}") - print(" ".join(d for d, _ in pick(a.model, a.family, a.vendor))) + flavors = {f.strip() for f in a.flavors.split(",") if f.strip()} + ops = {o.strip() for o in a.operations.split(",") if o.strip()} + p = plan(a.model, a.family, a.vendor, flavors, ops) + if not p: + raise SystemExit(f"no client covers flavors {sorted(flavors)} / operations {sorted(ops)}") + touched = [d for d, expr in p.items() if inject(Path(a.repo) / CLIENTS / d / "conftest.py", a.model, expr)] + for d, expr in p.items(): + print(f" {d:11s} <- {expr}") + print(f"injected {a.model} (flavors {','.join(sorted(flavors))}) into: {' '.join(touched) or '(already present)'}") + print(" ".join(p)) return 0