From ad45dbc5092735ad688a4ff9ef021b5bf8b405c8 Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Thu, 5 Nov 2026 01:52:48 -0600 Subject: [PATCH 1/8] feat(copilot): map workflow reasoning effort and context size Translate the reasoning_effort and context_size workflow step settings to --reasoning-effort and --context on both Copilot argv paths: prompt steps through build_exec_args() and command steps through the overridden dispatch_command(). The flags follow --model. Copilot CLI validates the values itself. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/integrations/copilot.md | 15 ++ .../integrations/copilot/__init__.py | 53 +++- .../integrations/test_integration_copilot.py | 249 +++++++++++++++++- 3 files changed, 315 insertions(+), 2 deletions(-) diff --git a/docs/integrations/copilot.md b/docs/integrations/copilot.md index b0a5b57686..a81fc91983 100644 --- a/docs/integrations/copilot.md +++ b/docs/integrations/copilot.md @@ -116,3 +116,18 @@ without interactive approval. Set `SPECKIT_COPILOT_ALLOW_ALL_TOOLS=0` to stop Spec Kit from adding this flag. The deprecated `SPECKIT_ALLOW_ALL_TOOLS` is accepted only as a fallback. A configured `--model` and requested `--output-format json` are forwarded. + +## Workflow Dispatch Settings + +Command and prompt steps can set +[`reasoning_effort` and `context_size`](../workflow-steps/types.md#reasoning-effort-and-context-size). +Copilot CLI dispatch passes them as native flags after `--model`, in both the +skills and commands layouts: + +| Step setting | Copilot CLI flag | Values in Copilot CLI 1.0.106 | +|---|---|---| +| `reasoning_effort` | `--reasoning-effort ` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | +| `context_size` | `--context ` | `default`, `long_context` | + +Spec Kit passes the values through unchanged. Copilot CLI rejects a value it +does not recognize, and the step fails like any other CLI error. diff --git a/src/specify_cli/integrations/copilot/__init__.py b/src/specify_cli/integrations/copilot/__init__.py index 6f373613bb..c7de7225bd 100644 --- a/src/specify_cli/integrations/copilot/__init__.py +++ b/src/specify_cli/integrations/copilot/__init__.py @@ -30,7 +30,13 @@ import typer from .._capture import run_captured -from ..base import IntegrationBase, IntegrationOption, SkillsIntegration +from ..base import ( + IntegrationBase, + IntegrationOption, + SkillsIntegration, + check_dispatch_settings, + requested_dispatch_settings, +) from ..manifest import IntegrationManifest _COPILOT_CORE_COMMANDS = { @@ -152,6 +158,9 @@ class CopilotIntegration(IntegrationBase): "extension": ".agent.md", } invoke_separator = "-" + # Translated by both receiving methods defined below: build_exec_args() + # for prompt steps and dispatch_command() for command steps. + supported_dispatch_settings = frozenset({"reasoning_effort", "context_size"}) CANONICAL_TO_NATIVE = { "session_start": "sessionStart", @@ -332,6 +341,32 @@ def _resolve_executable(self) -> str: override = os.environ.get(env_name, "").strip() return override if override else _copilot_executable() + def _dispatch_setting_args( + self, + *, + via: str, + reasoning_effort: str | None, + context_size: str | None, + ) -> list[str]: + """Gate per-step dispatch settings and return their Copilot CLI flags. + + *via* names the method that received the settings, as for + :func:`check_dispatch_settings`. Values pass through unchanged; the + CLI rejects levels or tiers it does not recognize. + """ + settings = requested_dispatch_settings( + reasoning_effort=reasoning_effort, context_size=context_size + ) + if not settings: + return [] + check_dispatch_settings(self, settings, via=via) + args: list[str] = [] + if reasoning_effort is not None: + args.extend(["--reasoning-effort", reasoning_effort]) + if context_size is not None: + args.extend(["--context", context_size]) + return args + def build_exec_args( self, prompt: str, @@ -341,8 +376,15 @@ def build_exec_args( integration_args: Sequence[str] | None = None, integration_options: Mapping[str, Any] | None = None, project_root: Path | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, ) -> list[str] | None: self.validate_runtime_config(integration_args, integration_options) + setting_args = self._dispatch_setting_args( + via="build_exec_args", + reasoning_effort=reasoning_effort, + context_size=context_size, + ) # GitHub Copilot CLI uses ``copilot -p "prompt"`` for # non-interactive mode. --yolo enables all permissions # (tools, paths, and URLs) so the agent can perform file @@ -356,6 +398,7 @@ def build_exec_args( args.append("--yolo") if model: args.extend(["--model", model]) + args.extend(setting_args) if output_json: args.extend(["--output-format", "json"]) return args @@ -389,6 +432,8 @@ def dispatch_command( integration_options: Mapping[str, Any] | None = None, output_json: bool | None = None, capture_max_bytes: int | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, ) -> dict[str, Any]: """Dispatch via ``--agent speckit.`` instead of slash-commands. @@ -407,6 +452,11 @@ def dispatch_command( output_json = not stream self.validate_runtime_config(integration_args, integration_options) + setting_args = self._dispatch_setting_args( + via="dispatch_command", + reasoning_effort=reasoning_effort, + context_size=context_size, + ) stem = command_name if stem.startswith("speckit."): @@ -438,6 +488,7 @@ def dispatch_command( cli_args.append("--yolo") if model: cli_args.extend(["--model", model]) + cli_args.extend(setting_args) if output_json: cli_args.extend(["--output-format", "json"]) diff --git a/tests/integrations/test_integration_copilot.py b/tests/integrations/test_integration_copilot.py index 796cdc3d97..8563150bfa 100644 --- a/tests/integrations/test_integration_copilot.py +++ b/tests/integrations/test_integration_copilot.py @@ -2,6 +2,8 @@ import json import os +import shutil +import subprocess from pathlib import Path from unittest.mock import patch @@ -10,8 +12,17 @@ import yaml from specify_cli.integrations import get_integration -from specify_cli.integrations.base import SkillsIntegration +from specify_cli.integrations.base import ( + DispatchSettingsError, + SkillsIntegration, + check_dispatch_settings, + effective_dispatch_settings, +) +from specify_cli.integrations.copilot import CopilotIntegration from specify_cli.integrations.manifest import IntegrationManifest +from specify_cli.workflows.base import StepContext, StepStatus +from specify_cli.workflows.step.command import CommandStep +from specify_cli.workflows.step.prompt import PromptStep REPO_ROOT = Path(__file__).resolve().parents[2] @@ -1257,3 +1268,239 @@ def test_extension_hook_skills_stay_model_invocable_after_upgrade(self, tmp_path assert _skill_frontmatter(skills_dir / "speckit-specify" / "SKILL.md")[ "disable-model-invocation" ] is True + + +class _LegacyBuilderCopilot(CopilotIntegration): + """Overrides only ``build_exec_args``, with the signature that predates + per-step dispatch settings.""" + + def build_exec_args( + self, + prompt, + *, + model=None, + output_json=True, + integration_args=None, + integration_options=None, + project_root=None, + ): + return super().build_exec_args( + prompt, + model=model, + output_json=output_json, + integration_args=integration_args, + integration_options=integration_options, + project_root=project_root, + ) + + +_COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) +_PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) + + +class TestCopilotDispatchSettings: + """Per-step ``reasoning_effort`` and ``context_size`` (#4916).""" + + _LAYOUTS = pytest.mark.parametrize( + ("skills_mode", "dispatch_head"), + [ + (True, ["copilot", "-p", "/speckit-plan body"]), + (False, ["copilot", "-p", "body", "--agent", "speckit.plan"]), + ], + ids=["skills", "commands"], + ) + + @pytest.fixture(autouse=True) + def _deterministic_argv(self, monkeypatch): + monkeypatch.setenv("SPECKIT_INTEGRATION_COPILOT_EXECUTABLE", "copilot") + monkeypatch.setenv("SPECKIT_COPILOT_ALLOW_ALL_TOOLS", "0") + + @pytest.fixture + def runs(self, monkeypatch): + calls = [] + + def fake_run(args, **kwargs): + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + @pytest.fixture + def no_runs(self, monkeypatch): + def fail(*args, **kwargs): + raise AssertionError("the Copilot CLI must not start") + + monkeypatch.setattr(subprocess, "run", fail) + + @staticmethod + def _copilot(skills_mode=True, cls=CopilotIntegration): + copilot = cls() + copilot._skills_mode = skills_mode + return copilot + + @pytest.mark.parametrize("via", ["dispatch_command", "build_exec_args"]) + def test_both_receiving_methods_declare_both_settings(self, via): + assert effective_dispatch_settings( + get_integration("copilot"), via=via + ) == frozenset({"reasoning_effort", "context_size"}) + + @pytest.mark.parametrize("skills_mode", [True, False], ids=["skills", "commands"]) + def test_build_exec_args_emits_flags_after_model(self, skills_mode): + args = self._copilot(skills_mode).build_exec_args( + "do it", + model="gpt-5", + reasoning_effort="high", + context_size="long_context", + ) + assert args == [ + "copilot", "-p", "do it", + "--model", "gpt-5", + "--reasoning-effort", "high", + "--context", "long_context", + "--output-format", "json", + ] + + @pytest.mark.parametrize( + ("settings", "flags"), + [ + ({"reasoning_effort": "low"}, ["--reasoning-effort", "low"]), + ({"context_size": "default"}, ["--context", "default"]), + ], + ids=["reasoning_effort", "context_size"], + ) + def test_build_exec_args_without_model_emits_only_requested_flags( + self, settings, flags + ): + args = self._copilot().build_exec_args("do it", **settings) + assert args == ["copilot", "-p", "do it", *flags, "--output-format", "json"] + + @_LAYOUTS + def test_dispatch_command_emits_flags_after_model( + self, runs, skills_mode, dispatch_head + ): + result = self._copilot(skills_mode).dispatch_command( + "speckit.plan", + "body", + model="gpt-5", + stream=False, + reasoning_effort="xhigh", + context_size="long_context", + ) + assert result["exit_code"] == 0 + assert runs == [ + [ + *dispatch_head, + "--model", "gpt-5", + "--reasoning-effort", "xhigh", + "--context", "long_context", + "--output-format", "json", + ] + ] + + @_LAYOUTS + def test_unset_settings_leave_argv_unchanged( + self, runs, skills_mode, dispatch_head + ): + copilot = self._copilot(skills_mode) + baseline = [ + "copilot", "-p", "do it", "--model", "gpt-5", "--output-format", "json" + ] + assert copilot.build_exec_args("do it", model="gpt-5") == baseline + assert copilot.build_exec_args( + "do it", model="gpt-5", reasoning_effort=None, context_size=None + ) == baseline + + copilot.dispatch_command("speckit.plan", "body", model="gpt-5", stream=False) + copilot.dispatch_command( + "speckit.plan", + "body", + model="gpt-5", + stream=False, + reasoning_effort=None, + context_size=None, + ) + expected = [*dispatch_head, "--model", "gpt-5", "--output-format", "json"] + assert runs == [expected, expected] + + def test_values_pass_through_for_the_cli_to_validate(self): + args = self._copilot().build_exec_args( + "do it", reasoning_effort="bogus", context_size="huge" + ) + assert args[3:7] == ["--reasoning-effort", "bogus", "--context", "huge"] + + @pytest.mark.usefixtures("no_runs") + @pytest.mark.parametrize( + ("setting", "value"), + [ + ("reasoning_effort", ""), + ("context_size", ""), + ("reasoning_effort", "--yolo"), + ("context_size", "long context"), + ], + ) + def test_direct_calls_reject_malformed_values(self, setting, value): + copilot = self._copilot() + with pytest.raises(DispatchSettingsError) as built: + copilot.build_exec_args("do it", **{setting: value}) + with pytest.raises(DispatchSettingsError) as dispatched: + copilot.dispatch_command("speckit.plan", stream=False, **{setting: value}) + assert built.value.setting == setting + assert dispatched.value.setting == setting + + def test_legacy_builder_subclass_keeps_the_command_path(self, runs): + legacy = self._copilot(cls=_LegacyBuilderCopilot) + settings = {"reasoning_effort": "high", "context_size": "long_context"} + + assert effective_dispatch_settings( + legacy, via="dispatch_command" + ) == frozenset(settings) + check_dispatch_settings(legacy, settings, via="dispatch_command") + legacy.dispatch_command("speckit.plan", "body", stream=False, **settings) + assert runs == [ + [ + "copilot", "-p", "/speckit-plan body", + "--reasoning-effort", "high", + "--context", "long_context", + "--output-format", "json", + ] + ] + + def test_legacy_builder_subclass_rejects_the_prompt_path(self): + legacy = self._copilot(cls=_LegacyBuilderCopilot) + + assert effective_dispatch_settings(legacy, via="build_exec_args") == frozenset() + with pytest.raises( + DispatchSettingsError, + match="does not support per-step 'reasoning_effort'", + ) as excinfo: + check_dispatch_settings( + legacy, {"reasoning_effort": "high"}, via="build_exec_args" + ) + assert excinfo.value.setting == "reasoning_effort" + + @pytest.mark.parametrize( + ("step", "config", "prompt"), + [(*_COMMAND_STEP, "/speckit-plan body"), (*_PROMPT_STEP, "Summarise")], + ids=["command", "prompt"], + ) + def test_workflow_steps_dispatch_settings( + self, monkeypatch, tmp_path, runs, step, config, prompt + ): + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + result = step().execute( + { + "id": "ask", + **config, + "reasoning_effort": "high", + "context_size": "long_context", + }, + StepContext(default_integration="copilot", project_root=str(tmp_path)), + ) + assert result.status == StepStatus.COMPLETED, result.error + assert result.output["reasoning_effort"] == "high" + assert result.output["context_size"] == "long_context" + (args,) = runs + assert args[1:] == [ + "-p", prompt, "--reasoning-effort", "high", "--context", "long_context" + ] From 3c1876384b5ee9ceff1cc3636c9a15a58dc4e8e0 Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Thu, 5 Nov 2026 01:52:48 -0600 Subject: [PATCH 2/8] feat(codex): map workflow reasoning effort and context size Translate reasoning_effort to -c model_reasoning_effort="" and context_size to -c model_context_window= after --model. context_size must be a positive whole number no larger than the i64 range that Codex accepts. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/integrations/codex.md | 19 ++ .../integrations/codex/__init__.py | 39 +++- tests/integrations/test_integration_codex.py | 173 ++++++++++++++++++ 3 files changed, 230 insertions(+), 1 deletion(-) diff --git a/docs/integrations/codex.md b/docs/integrations/codex.md index d37b6901f5..967f11db2b 100644 --- a/docs/integrations/codex.md +++ b/docs/integrations/codex.md @@ -36,3 +36,22 @@ See [runtime events](cli.md#runtime-events). Workflow steps use `codex exec `, with `--model` when configured and `--json` when structured output is requested. Codex dev-mode registration writes skill files rather than symlinking them. + +## Workflow Dispatch Settings + +Command and prompt steps can set +[`reasoning_effort` and `context_size`](../workflow-steps/types.md#reasoning-effort-and-context-size). +Codex dispatch passes them as configuration overrides after `--model`: + +| Step setting | Codex CLI argument | Values | +|---|---|---| +| `reasoning_effort` | `-c model_reasoning_effort=""` | A level the selected model supports, such as `low`, `medium`, `high`, or `xhigh` | +| `context_size` | `-c model_context_window=` | A token count: a positive whole number without leading zeros, at most `9223372036854775807` | + +Spec Kit passes `reasoning_effort` through unchanged; Codex CLI 0.165.0 +forwards it to the model without checking it. A `context_size` that is not a +valid token count fails the step before Codex starts. + +`context_size` overrides the context window that Codex assumes for the model. +Codex clamps the value to the model's maximum, and a value below the model's +default shrinks the usable context. diff --git a/src/specify_cli/integrations/codex/__init__.py b/src/specify_cli/integrations/codex/__init__.py index d88628b3ff..a1b9446f29 100644 --- a/src/specify_cli/integrations/codex/__init__.py +++ b/src/specify_cli/integrations/codex/__init__.py @@ -5,12 +5,22 @@ """ from __future__ import annotations +import re from pathlib import Path from collections.abc import Mapping, Sequence from typing import Any -from ..base import IntegrationOption, SkillsIntegration +from ..base import ( + IntegrationOption, + SkillsIntegration, + check_dispatch_settings, + requested_dispatch_settings, +) + +# ``model_context_window`` is a TOML integer that Codex reads as an i64. +_CONTEXT_WINDOW = re.compile(r"[1-9][0-9]*") +_MAX_CONTEXT_WINDOW = 2**63 - 1 class CodexIntegration(SkillsIntegration): @@ -32,6 +42,7 @@ class CodexIntegration(SkillsIntegration): } dev_no_symlink = True multi_install_safe = True + supported_dispatch_settings = frozenset({"reasoning_effort", "context_size"}) CANONICAL_TO_NATIVE = { "session_start": "SessionStart", @@ -53,19 +64,45 @@ def build_exec_args( integration_args: Sequence[str] | None = None, integration_options: Mapping[str, Any] | None = None, project_root: Path | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, ) -> list[str] | None: # Codex uses ``codex exec "prompt"`` for non-interactive mode. # Resolve argv[0] via the shared executable resolver so operators can # override the binary with SPECKIT_INTEGRATION_CODEX_EXECUTABLE. self.validate_runtime_config(integration_args, integration_options) + settings = requested_dispatch_settings( + reasoning_effort=reasoning_effort, context_size=context_size + ) + if settings: + check_dispatch_settings(self, settings, via="build_exec_args") args: list[str] = [self._resolve_executable(), "exec", prompt] self._apply_extra_args_env_var(args) if model: args.extend(["--model", model]) + # ``-c`` values are parsed as TOML. The checked value cannot contain + # quotes, backslashes, or control characters, so it forms a valid + # basic string as-is. + if reasoning_effort is not None: + args.extend(["-c", f'model_reasoning_effort="{reasoning_effort}"']) + if context_size is not None: + args.extend(["-c", f"model_context_window={context_size}"]) if output_json: args.append("--json") return args + def validate_dispatch_setting(self, name: str, value: str) -> None: + """Require ``context_size`` to be a token count Codex can parse.""" + if name == "context_size" and ( + _CONTEXT_WINDOW.fullmatch(value) is None + or int(value) > _MAX_CONTEXT_WINDOW + ): + raise ValueError( + "Codex 'context_size' must be a positive integer token count " + "(model_context_window) without leading zeros, at most " + f"{_MAX_CONTEXT_WINDOW}; got {value!r}." + ) + @classmethod def options(cls) -> list[IntegrationOption]: opts = super().options() diff --git a/tests/integrations/test_integration_codex.py b/tests/integrations/test_integration_codex.py index 9a4dc5442b..71aa412b4e 100644 --- a/tests/integrations/test_integration_codex.py +++ b/tests/integrations/test_integration_codex.py @@ -1,7 +1,20 @@ """Tests for CodexIntegration.""" +import shutil +import subprocess +import tomllib + +import pytest + from specify_cli.integrations import get_integration +from specify_cli.integrations.base import ( + DispatchSettingsError, + effective_dispatch_settings, +) from specify_cli.integrations.manifest import IntegrationManifest +from specify_cli.workflows.base import StepContext, StepStatus +from specify_cli.workflows.step.command import CommandStep +from specify_cli.workflows.step.prompt import PromptStep from .test_integration_base_skills import SkillsIntegrationTests @@ -191,3 +204,163 @@ def test_hook_note_when_instruction_is_final_line_without_newline(self): assert "For each executable hook" not in lines[note_line_idx], ( "Note and instruction must not be on the same line" ) + + +_COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) +_PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) + + +class TestCodexDispatchSettings: + """Per-step ``reasoning_effort`` and ``context_size`` (#4916).""" + + _CODEX_RULE = "positive integer token count (model_context_window)" + + @pytest.fixture + def runs(self, monkeypatch): + calls = [] + + def fake_run(args, **kwargs): + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + @pytest.mark.parametrize("via", ["dispatch_command", "build_exec_args"]) + def test_build_exec_args_declares_both_settings(self, via): + assert effective_dispatch_settings( + get_integration("codex"), via=via + ) == frozenset({"reasoning_effort", "context_size"}) + + def test_settings_become_config_overrides_after_model(self): + args = get_integration("codex").build_exec_args( + "do it", model="gpt-5", reasoning_effort="high", context_size="200000" + ) + assert args == [ + "codex", "exec", "do it", + "--model", "gpt-5", + "-c", 'model_reasoning_effort="high"', + "-c", "model_context_window=200000", + "--json", + ] + + @pytest.mark.parametrize("value", ["high", "xhigh", "v2.max-ultra_1"]) + def test_reasoning_effort_is_embedded_as_a_toml_string(self, value): + args = get_integration("codex").build_exec_args( + "do it", reasoning_effort=value, output_json=False + ) + assert args[-2] == "-c" + assert tomllib.loads(args[-1]) == {"model_reasoning_effort": value} + + def test_reasoning_effort_values_are_left_to_codex(self): + args = get_integration("codex").build_exec_args( + "do it", reasoning_effort="bogus" + ) + assert args[-3:] == ["-c", 'model_reasoning_effort="bogus"', "--json"] + + @pytest.mark.parametrize("value", ["1", "200000", "9223372036854775807"]) + def test_context_size_accepts_canonical_i64_token_counts(self, value): + args = get_integration("codex").build_exec_args( + "do it", context_size=value, output_json=False + ) + assert args[-2:] == ["-c", f"model_context_window={value}"] + assert tomllib.loads(args[-1]) == {"model_context_window": int(value)} + + @pytest.mark.parametrize( + ("value", "reason"), + [ + ("0", _CODEX_RULE), + ("01", _CODEX_RULE), + ("9223372036854775808", _CODEX_RULE), + ("1e5", _CODEX_RULE), + ("long_context", _CODEX_RULE), + ("-5", "starting with a letter or digit"), + ], + ids=["zero", "leading-zero", "above-i64", "exponent", "tier-name", "negative"], + ) + def test_context_size_rejects_other_values(self, value, reason): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("codex").build_exec_args("do it", context_size=value) + assert excinfo.value.setting == "context_size" + assert reason in str(excinfo.value) + + def test_unset_settings_leave_argv_unchanged(self): + codex = get_integration("codex") + baseline = ["codex", "exec", "do it", "--model", "gpt-5", "--json"] + assert codex.build_exec_args("do it", model="gpt-5") == baseline + assert codex.build_exec_args( + "do it", model="gpt-5", reasoning_effort=None, context_size=None + ) == baseline + + def test_dispatch_command_forwards_settings(self, runs): + get_integration("codex").dispatch_command( + "speckit.plan", + "body", + stream=False, + reasoning_effort="low", + context_size="128000", + ) + (args,) = runs + assert args[1:] == [ + "exec", "$speckit-plan body", + "-c", 'model_reasoning_effort="low"', + "-c", "model_context_window=128000", + "--json", + ] + + def test_dispatch_command_rejects_before_the_cli_starts(self, runs): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("codex").dispatch_command( + "speckit.plan", stream=False, context_size="long_context" + ) + assert excinfo.value.setting == "context_size" + assert self._CODEX_RULE in str(excinfo.value) + assert runs == [] + + @staticmethod + def _execute(monkeypatch, tmp_path, step, config, **settings): + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + return step().execute( + {"id": "ask", **config, **settings}, + StepContext(default_integration="codex", project_root=str(tmp_path)), + ) + + @pytest.mark.parametrize( + ("step", "config", "prompt"), + [(*_COMMAND_STEP, "$speckit-plan body"), (*_PROMPT_STEP, "Summarise")], + ids=["command", "prompt"], + ) + def test_workflow_steps_dispatch_settings( + self, monkeypatch, tmp_path, runs, step, config, prompt + ): + result = self._execute( + monkeypatch, + tmp_path, + step, + config, + reasoning_effort="high", + context_size="200000", + ) + assert result.status == StepStatus.COMPLETED, result.error + assert result.output["reasoning_effort"] == "high" + assert result.output["context_size"] == "200000" + (args,) = runs + assert args[1:] == [ + "exec", prompt, + "-c", 'model_reasoning_effort="high"', + "-c", "model_context_window=200000", + ] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + def test_workflow_steps_reject_invalid_context_size( + self, monkeypatch, tmp_path, runs, step, config + ): + result = self._execute( + monkeypatch, tmp_path, step, config, context_size="long_context" + ) + assert result.status == StepStatus.FAILED + assert result.output["dispatched"] is False + assert self._CODEX_RULE in result.error + assert runs == [] From c3aa7ed4f46e66800025e59d920fad1ed3254627 Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Thu, 5 Nov 2026 01:52:48 -0600 Subject: [PATCH 3/8] feat(claude): map workflow reasoning effort to --effort Claude Code only warns about an unknown --effort value and then runs at its default effort, so Spec Kit checks the value against Claude's documented levels (low, medium, high, xhigh, max) before dispatch. The error names SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS as an escape hatch for newer levels. context_size is not mapped; its error points to choosing a long-context model. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/integrations/claude.md | 15 ++ .../integrations/claude/__init__.py | 64 +++++- tests/integrations/test_integration_claude.py | 211 +++++++++++++++++- 3 files changed, 287 insertions(+), 3 deletions(-) diff --git a/docs/integrations/claude.md b/docs/integrations/claude.md index 5f0fa5eacc..3bb067f1fb 100644 --- a/docs/integrations/claude.md +++ b/docs/integrations/claude.md @@ -69,3 +69,18 @@ See [runtime events](cli.md#runtime-events). Workflow steps use `claude -p `, with `--model` when configured and `--output-format json` when structured output is requested. This adapter does not add a permission-bypass flag itself. + +## Workflow Dispatch Settings + +Command and prompt steps can set +[`reasoning_effort`](../workflow-steps/types.md#reasoning-effort-and-context-size), +which Claude Code dispatch passes as `--effort `. Claude Code 2.1.318 +accepts `low`, `medium`, `high`, `xhigh`, and `max`. It ignores any other +value with only a warning, so Spec Kit rejects other values before Claude Code +starts. To pass a level that a newer Claude Code release adds, set +`SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort "` instead of +`reasoning_effort`; the environment variable applies to every Claude Code +invocation, not just one step. + +Spec Kit does not map `context_size` for Claude Code, so a step that sets it +fails. Choose a long-context model with the step's `model` field instead. diff --git a/src/specify_cli/integrations/claude/__init__.py b/src/specify_cli/integrations/claude/__init__.py index e2de919b8c..628f776346 100644 --- a/src/specify_cli/integrations/claude/__init__.py +++ b/src/specify_cli/integrations/claude/__init__.py @@ -2,13 +2,24 @@ from __future__ import annotations +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any -from ..base import IntegrationOption, SkillsIntegration +from ..base import ( + IntegrationOption, + SkillsIntegration, + check_dispatch_settings, + requested_dispatch_settings, +) from ..manifest import IntegrationManifest from ..._utils import dump_frontmatter +# ``--effort`` levels accepted by Claude Code 2.1.318. The CLI only warns about +# any other value and runs with its default effort, so per-step +# ``reasoning_effort`` is limited to these. +EFFORT_LEVELS: tuple[str, ...] = ("low", "medium", "high", "xhigh", "max") + # Mapping of command template stem → argument-hint text shown inline # when a user invokes the slash command in Claude Code. ARGUMENT_HINTS: dict[str, str] = { @@ -65,6 +76,7 @@ class ClaudeIntegration(SkillsIntegration): } events_config_file = ".claude/settings.json" events_format = "json-nested" + supported_dispatch_settings = frozenset({"reasoning_effort"}) # Set by ``setup()`` from ``--no-model-invocation``; applies to core # command skills only, since mandatory extension hooks rely on the model @@ -91,6 +103,56 @@ def options(cls) -> list[IntegrationOption]: ) return opts + def build_exec_args( + self, + prompt: str, + *, + model: str | None = None, + output_json: bool = True, + integration_args: Sequence[str] | None = None, + integration_options: Mapping[str, Any] | None = None, + project_root: Path | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, + ) -> list[str] | None: + """Build ``claude -p`` arguments, adding ``--effort`` when requested.""" + # super() validates the runtime config, once, before the gate runs, so + # its errors come first, as in dispatch_command(). + args = super().build_exec_args( + prompt, + model=model, + output_json=output_json, + integration_args=integration_args, + integration_options=integration_options, + project_root=project_root, + ) + settings = requested_dispatch_settings( + reasoning_effort=reasoning_effort, context_size=context_size + ) + if settings: + check_dispatch_settings(self, settings, via="build_exec_args") + if args is not None and reasoning_effort is not None: + args.extend(["--effort", reasoning_effort]) + return args + + def validate_dispatch_setting(self, name: str, value: str) -> None: + """Reject effort levels that Claude Code would silently ignore.""" + if name == "reasoning_effort" and value not in EFFORT_LEVELS: + raise ValueError( + "Claude Code 'reasoning_effort' must be one of " + f"{', '.join(EFFORT_LEVELS)}; got {value!r}. To pass a newer " + 'level, set SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort " ' + "instead of 'reasoning_effort'." + ) + + def dispatch_setting_hint(self, setting: str) -> str | None: + if setting == "context_size": + return ( + "Choose a long-context model with the step's 'model' field " + "instead." + ) + return None + def setup( self, project_root: Path, diff --git a/tests/integrations/test_integration_claude.py b/tests/integrations/test_integration_claude.py index f275737075..cb6788ca15 100644 --- a/tests/integrations/test_integration_claude.py +++ b/tests/integrations/test_integration_claude.py @@ -2,6 +2,8 @@ import json import os +import shutil +import subprocess from pathlib import Path from unittest.mock import patch @@ -9,9 +11,22 @@ import yaml from specify_cli.integrations import INTEGRATION_REGISTRY, get_integration -from specify_cli.integrations.base import IntegrationBase, SkillsIntegration -from specify_cli.integrations.claude import ARGUMENT_HINTS, FORK_CONTEXT_COMMANDS +from specify_cli.integrations.base import ( + DispatchSettingsError, + IntegrationBase, + SkillsIntegration, + check_dispatch_settings, + effective_dispatch_settings, +) +from specify_cli.integrations.claude import ( + ARGUMENT_HINTS, + EFFORT_LEVELS, + FORK_CONTEXT_COMMANDS, +) from specify_cli.integrations.manifest import IntegrationManifest +from specify_cli.workflows.base import StepContext, StepStatus +from specify_cli.workflows.step.command import CommandStep +from specify_cli.workflows.step.prompt import PromptStep REPO_ROOT = Path(__file__).resolve().parents[2] @@ -1248,3 +1263,195 @@ def test_install_shared_infra_handles_directory_at_template_destination( assert real_template in text, ( "directory-at-template-dst path must surface in the skipped warning" ) + + +_COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) +_PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) + + +class TestClaudeDispatchSettings: + """Per-step ``reasoning_effort`` maps to ``--effort`` (#4916).""" + + @pytest.fixture + def runs(self, monkeypatch): + calls = [] + + def fake_run(args, **kwargs): + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + @pytest.mark.parametrize("via", ["dispatch_command", "build_exec_args"]) + def test_build_exec_args_owns_reasoning_effort_only(self, via): + from specify_cli.integrations.claude import ClaudeIntegration + + assert "build_exec_args" in vars(ClaudeIntegration) + assert effective_dispatch_settings( + ClaudeIntegration(), via=via + ) == frozenset({"reasoning_effort"}) + + def test_effort_levels_match_claude_code(self): + assert EFFORT_LEVELS == ("low", "medium", "high", "xhigh", "max") + + @pytest.mark.parametrize("level", ["low", "medium", "high", "xhigh", "max"]) + def test_every_effort_level_becomes_the_effort_flag(self, level): + claude = get_integration("claude") + check_dispatch_settings( + claude, {"reasoning_effort": level}, via="build_exec_args" + ) + assert claude.build_exec_args( + "do it", model="sonnet", reasoning_effort=level + ) == [ + "claude", "-p", "do it", + "--model", "sonnet", + "--output-format", "json", + "--effort", level, + ] + + def test_dispatch_command_passes_effort(self, runs): + get_integration("claude").dispatch_command( + "speckit.plan", "body", stream=False, reasoning_effort="max" + ) + (args,) = runs + assert args[1:] == [ + "-p", "/speckit-plan body", "--output-format", "json", "--effort", "max" + ] + + @pytest.mark.parametrize("level", ["bogus", "none", "minimal", "HIGH"]) + def test_unknown_effort_fails_before_the_cli_starts(self, runs, level): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("claude").dispatch_command( + "speckit.plan", stream=False, reasoning_effort=level + ) + assert runs == [] + assert excinfo.value.setting == "reasoning_effort" + message = str(excinfo.value) + assert "must be one of low, medium, high, xhigh, max" in message + assert 'SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort "' in message + + def test_direct_build_rejects_unknown_effort(self): + with pytest.raises( + DispatchSettingsError, match="SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS" + ) as excinfo: + get_integration("claude").build_exec_args( + "do it", reasoning_effort="bogus" + ) + assert excinfo.value.setting == "reasoning_effort" + + def test_context_size_is_rejected_with_a_model_hint(self, runs): + claude = get_integration("claude") + with pytest.raises(DispatchSettingsError) as excinfo: + claude.dispatch_command( + "speckit.plan", stream=False, context_size="long_context" + ) + assert runs == [] + assert excinfo.value.setting == "context_size" + message = str(excinfo.value) + assert "does not support per-step 'context_size'" in message + assert ( + "Choose a long-context model with the step's 'model' field instead." + in message + ) + assert claude.dispatch_setting_hint("reasoning_effort") is None + + def test_context_size_is_attributed_when_both_settings_are_set(self): + with pytest.raises(DispatchSettingsError, match="'model' field") as excinfo: + get_integration("claude").build_exec_args( + "do it", reasoning_effort="high", context_size="long_context" + ) + assert excinfo.value.setting == "context_size" + + def test_unset_settings_leave_argv_unchanged(self): + claude = get_integration("claude") + baseline = [ + "claude", "-p", "do it", "--model", "sonnet", "--output-format", "json" + ] + assert claude.build_exec_args("do it", model="sonnet") == baseline + assert claude.build_exec_args( + "do it", model="sonnet", reasoning_effort=None, context_size=None + ) == baseline + + def test_runtime_config_is_validated_once(self, monkeypatch): + from specify_cli.integrations.claude import ClaudeIntegration + + claude = ClaudeIntegration() + calls = [] + monkeypatch.setattr( + claude, + "validate_runtime_config", + lambda *args: calls.append(args), + ) + + claude.build_exec_args("do it", integration_args=["a"]) + assert calls == [(["a"], None)] + + calls.clear() + claude.build_exec_args( + "do it", integration_args=["a"], reasoning_effort="high" + ) + assert calls == [(["a"], None)] + + def test_runtime_config_errors_precede_setting_errors(self): + with pytest.raises(ValueError, match="'integration_args'"): + get_integration("claude").build_exec_args( + "do it", integration_args=["x"], reasoning_effort="bogus" + ) + + def test_extra_args_escape_hatch_passes_newer_levels(self, monkeypatch): + monkeypatch.setenv("SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS", "--effort ultra") + assert get_integration("claude").build_exec_args("do it") == [ + "claude", "-p", "do it", "--effort", "ultra", "--output-format", "json" + ] + + @staticmethod + def _execute(monkeypatch, tmp_path, step, config, **settings): + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + return step().execute( + {"id": "ask", **config, **settings}, + StepContext(default_integration="claude", project_root=str(tmp_path)), + ) + + @pytest.mark.parametrize( + ("step", "config", "prompt"), + [(*_COMMAND_STEP, "/speckit-plan body"), (*_PROMPT_STEP, "Summarise")], + ids=["command", "prompt"], + ) + def test_workflow_steps_dispatch_effort( + self, monkeypatch, tmp_path, runs, step, config, prompt + ): + result = self._execute( + monkeypatch, tmp_path, step, config, reasoning_effort="high" + ) + assert result.status == StepStatus.COMPLETED, result.error + assert result.output["reasoning_effort"] == "high" + assert result.output["context_size"] is None + (args,) = runs + assert args[1:] == ["-p", prompt, "--effort", "high"] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ( + {"reasoning_effort": "bogus"}, + 'SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort "', + ), + ( + {"context_size": "long_context"}, + "Choose a long-context model with the step's 'model' field", + ), + ], + ids=["unknown-effort", "context-size"], + ) + def test_workflow_steps_fail_before_the_cli_starts( + self, monkeypatch, tmp_path, runs, step, config, settings, expected + ): + result = self._execute(monkeypatch, tmp_path, step, config, **settings) + assert result.status == StepStatus.FAILED + assert result.output["dispatched"] is False + assert expected in result.error + assert runs == [] From f32b277b53988f8d138e8d740ae742b8ebde210a Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Thu, 5 Nov 2026 01:52:48 -0600 Subject: [PATCH 4/8] docs(workflows): list built-in reasoning effort and context size support Add the support table for GitHub Copilot, Codex CLI, and Claude Code to the workflow step reference and the shared CLI dispatch guide. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/integrations/cli.md | 4 +++- docs/workflow-steps/types.md | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/integrations/cli.md b/docs/integrations/cli.md index d67252af5d..91ccf0f07b 100644 --- a/docs/integrations/cli.md +++ b/docs/integrations/cli.md @@ -490,7 +490,9 @@ The [`reasoning_effort` and `context_size`](../workflow-steps/types.md#reasoning settings of `command` and `prompt` steps instead follow one shared contract. Spec Kit checks each value and passes it only to an integration that declares and translates that setting; with any other integration, the step fails before -the CLI runs. Adapter authors opt in as described in the +the CLI runs. GitHub Copilot and Codex CLI translate both settings, and Claude +Code translates `reasoning_effort`; the agent guides describe each mapping. +Adapter authors opt in as described in the [integration API reference](api.md#per-step-dispatch-settings). ## FAQ diff --git a/docs/workflow-steps/types.md b/docs/workflow-steps/types.md index 9a22906ce0..32d5434671 100644 --- a/docs/workflow-steps/types.md +++ b/docs/workflow-steps/types.md @@ -32,6 +32,7 @@ Invoke an installed Spec Kit command by name via the integration CLI: args: "{{ inputs.spec }}" integration: claude # Optional: override workflow default model: "claude-sonnet-4-20250514" # Optional: override model + reasoning_effort: "high" # Optional: requires integration support ``` For positional arguments and named runtime options, see @@ -254,8 +255,17 @@ integration's CLI, and an integration can reject a value before the CLI runs. for it. Otherwise the step fails before the CLI runs, with an error that names the setting, its source (the step or the workflow default), and any integrations that support it. Workflow defaults are not exempt: a default fails -each inheriting step whose integration does not support it. No built-in -integration supports these settings yet; see +each inheriting step whose integration does not support it. These built-in +integrations support the settings: + +| Integration | `reasoning_effort` | `context_size` | +| --- | --- | --- | +| [GitHub Copilot](../integrations/copilot.md#workflow-dispatch-settings) | Yes | Yes | +| [Codex CLI](../integrations/codex.md#workflow-dispatch-settings) | Yes | Yes | +| [Claude Code](../integrations/claude.md#workflow-dispatch-settings) | Yes | No | + +Other built-in integrations support neither setting. Each integration's guide +describes its CLI arguments and accepted values; see [Workflow CLI dispatch](../integrations/cli.md#workflow-cli-dispatch) for how integrations opt in. From b3116ff78834de6722d2cf1182a84b0199bc406e Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Thu, 5 Nov 2026 02:47:13 -0600 Subject: [PATCH 5/8] fix(integrations): omit rejected values from Claude and Codex errors A dispatch setting can resolve from captured agent output, and a step error is printed and written to log.jsonl, where captured text must never appear. Claude Code's reasoning_effort check and Codex's context_size check repeated the rejected value, so captured text that passes the host's value pattern reached both. The errors now state the setting and its rule without the value. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../integrations/claude/__init__.py | 9 +++-- .../integrations/codex/__init__.py | 7 +++- tests/integrations/test_integration_claude.py | 40 ++++++++++++++++++- tests/integrations/test_integration_codex.py | 40 ++++++++++++++++++- 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/specify_cli/integrations/claude/__init__.py b/src/specify_cli/integrations/claude/__init__.py index 628f776346..641480fe82 100644 --- a/src/specify_cli/integrations/claude/__init__.py +++ b/src/specify_cli/integrations/claude/__init__.py @@ -136,12 +136,15 @@ def build_exec_args( return args def validate_dispatch_setting(self, name: str, value: str) -> None: - """Reject effort levels that Claude Code would silently ignore.""" + """Reject effort levels that Claude Code would silently ignore. + + The value can be captured agent output, so the error does not repeat it. + """ if name == "reasoning_effort" and value not in EFFORT_LEVELS: raise ValueError( "Claude Code 'reasoning_effort' must be one of " - f"{', '.join(EFFORT_LEVELS)}; got {value!r}. To pass a newer " - 'level, set SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort " ' + f"{', '.join(EFFORT_LEVELS)}. To pass a newer level, set " + 'SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort " ' "instead of 'reasoning_effort'." ) diff --git a/src/specify_cli/integrations/codex/__init__.py b/src/specify_cli/integrations/codex/__init__.py index a1b9446f29..d346ff3a59 100644 --- a/src/specify_cli/integrations/codex/__init__.py +++ b/src/specify_cli/integrations/codex/__init__.py @@ -92,7 +92,10 @@ def build_exec_args( return args def validate_dispatch_setting(self, name: str, value: str) -> None: - """Require ``context_size`` to be a token count Codex can parse.""" + """Require ``context_size`` to be a token count Codex can parse. + + The value can be captured agent output, so the error does not repeat it. + """ if name == "context_size" and ( _CONTEXT_WINDOW.fullmatch(value) is None or int(value) > _MAX_CONTEXT_WINDOW @@ -100,7 +103,7 @@ def validate_dispatch_setting(self, name: str, value: str) -> None: raise ValueError( "Codex 'context_size' must be a positive integer token count " "(model_context_window) without leading zeros, at most " - f"{_MAX_CONTEXT_WINDOW}; got {value!r}." + f"{_MAX_CONTEXT_WINDOW}." ) @classmethod diff --git a/tests/integrations/test_integration_claude.py b/tests/integrations/test_integration_claude.py index cb6788ca15..5cc056d1e5 100644 --- a/tests/integrations/test_integration_claude.py +++ b/tests/integrations/test_integration_claude.py @@ -1267,6 +1267,9 @@ def test_install_shared_infra_handles_directory_at_template_destination( _COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) _PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) +# Captured agent output that passes the host's value pattern, so only Claude's +# own rule rejects it. +_SECRET_OUTPUT = "FAKEsecret0123456789" class TestClaudeDispatchSettings: @@ -1340,6 +1343,15 @@ def test_direct_build_rejects_unknown_effort(self): ) assert excinfo.value.setting == "reasoning_effort" + def test_rejected_effort_is_not_echoed(self): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("claude").build_exec_args( + "do it", reasoning_effort=_SECRET_OUTPUT + ) + message = str(excinfo.value) + assert _SECRET_OUTPUT not in message + assert "must be one of low, medium, high, xhigh, max." in message + def test_context_size_is_rejected_with_a_model_hint(self, runs): claude = get_integration("claude") with pytest.raises(DispatchSettingsError) as excinfo: @@ -1406,11 +1418,15 @@ def test_extra_args_escape_hatch_passes_newer_levels(self, monkeypatch): ] @staticmethod - def _execute(monkeypatch, tmp_path, step, config, **settings): + def _execute(monkeypatch, tmp_path, step, config, *, steps=None, **settings): monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") return step().execute( {"id": "ask", **config, **settings}, - StepContext(default_integration="claude", project_root=str(tmp_path)), + StepContext( + default_integration="claude", + project_root=str(tmp_path), + steps=steps or {}, + ), ) @pytest.mark.parametrize( @@ -1455,3 +1471,23 @@ def test_workflow_steps_fail_before_the_cli_starts( assert result.output["dispatched"] is False assert expected in result.error assert runs == [] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + def test_workflow_steps_do_not_echo_captured_output( + self, monkeypatch, tmp_path, runs, step, config + ): + result = self._execute( + monkeypatch, + tmp_path, + step, + config, + steps={"probe": {"output": {"stdout": _SECRET_OUTPUT}}}, + reasoning_effort="{{ steps.probe.output.stdout }}", + ) + assert result.status == StepStatus.FAILED + assert _SECRET_OUTPUT not in result.error + assert "must be one of low, medium, high, xhigh, max." in result.error + assert result.error.endswith(" (set by step)") + assert runs == [] diff --git a/tests/integrations/test_integration_codex.py b/tests/integrations/test_integration_codex.py index 71aa412b4e..abf4cb289c 100644 --- a/tests/integrations/test_integration_codex.py +++ b/tests/integrations/test_integration_codex.py @@ -208,6 +208,9 @@ def test_hook_note_when_instruction_is_final_line_without_newline(self): _COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) _PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) +# Captured agent output that passes the host's value pattern, so only Codex's +# own rule rejects it. +_SECRET_OUTPUT = "FAKEsecret0123456789" class TestCodexDispatchSettings: @@ -284,6 +287,15 @@ def test_context_size_rejects_other_values(self, value, reason): assert excinfo.value.setting == "context_size" assert reason in str(excinfo.value) + def test_rejected_context_size_is_not_echoed(self): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("codex").build_exec_args( + "do it", context_size=_SECRET_OUTPUT + ) + message = str(excinfo.value) + assert _SECRET_OUTPUT not in message + assert self._CODEX_RULE in message + def test_unset_settings_leave_argv_unchanged(self): codex = get_integration("codex") baseline = ["codex", "exec", "do it", "--model", "gpt-5", "--json"] @@ -318,11 +330,15 @@ def test_dispatch_command_rejects_before_the_cli_starts(self, runs): assert runs == [] @staticmethod - def _execute(monkeypatch, tmp_path, step, config, **settings): + def _execute(monkeypatch, tmp_path, step, config, *, steps=None, **settings): monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") return step().execute( {"id": "ask", **config, **settings}, - StepContext(default_integration="codex", project_root=str(tmp_path)), + StepContext( + default_integration="codex", + project_root=str(tmp_path), + steps=steps or {}, + ), ) @pytest.mark.parametrize( @@ -364,3 +380,23 @@ def test_workflow_steps_reject_invalid_context_size( assert result.output["dispatched"] is False assert self._CODEX_RULE in result.error assert runs == [] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + def test_workflow_steps_do_not_echo_captured_output( + self, monkeypatch, tmp_path, runs, step, config + ): + result = self._execute( + monkeypatch, + tmp_path, + step, + config, + steps={"probe": {"output": {"stdout": _SECRET_OUTPUT}}}, + context_size="{{ steps.probe.output.stdout }}", + ) + assert result.status == StepStatus.FAILED + assert _SECRET_OUTPUT not in result.error + assert self._CODEX_RULE in result.error + assert result.error.endswith(" (set by step)") + assert runs == [] From 0f2e4971cedf62f8219c5f99aec5be04d6538116 Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Fri, 6 Nov 2026 15:24:42 -0600 Subject: [PATCH 6/8] fix(copilot): validate dispatch setting values before dispatch Copilot CLI rejects an unknown --reasoning-effort level or --context tier with an error that repeats the value. A setting can resolve from captured agent output, so a step that does not capture could print captured text to the terminal. Spec Kit now accepts only the values that Copilot CLI 1.0.106 lists, and fails the step before the CLI starts. The error names the accepted values and the SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS escape hatch, but not the value. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/integrations/copilot.md | 9 +- .../integrations/copilot/__init__.py | 35 ++++- .../integrations/test_integration_copilot.py | 143 +++++++++++++++++- 3 files changed, 177 insertions(+), 10 deletions(-) diff --git a/docs/integrations/copilot.md b/docs/integrations/copilot.md index a81fc91983..12075e9b82 100644 --- a/docs/integrations/copilot.md +++ b/docs/integrations/copilot.md @@ -129,5 +129,10 @@ skills and commands layouts: | `reasoning_effort` | `--reasoning-effort ` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | | `context_size` | `--context ` | `default`, `long_context` | -Spec Kit passes the values through unchanged. Copilot CLI rejects a value it -does not recognize, and the step fails like any other CLI error. +Spec Kit accepts only these values. Any other value fails the step before +Copilot CLI starts, because Copilot CLI repeats a rejected value in its error +and the value can come from captured output. To pass a value that a newer +Copilot CLI release adds, set +`SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--reasoning-effort "` or +`SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--context "` instead of the +step setting. diff --git a/src/specify_cli/integrations/copilot/__init__.py b/src/specify_cli/integrations/copilot/__init__.py index c7de7225bd..205c25407a 100644 --- a/src/specify_cli/integrations/copilot/__init__.py +++ b/src/specify_cli/integrations/copilot/__init__.py @@ -51,6 +51,15 @@ "tasks", } +# ``--reasoning-effort`` levels and ``--context`` tiers accepted by GitHub +# Copilot CLI 1.0.106. The CLI rejects any other value with an error that +# repeats it, and a value can be captured agent output, so per-step +# ``reasoning_effort`` and ``context_size`` are limited to these. +REASONING_EFFORT_LEVELS: tuple[str, ...] = ( + "none", "minimal", "low", "medium", "high", "xhigh", "max" +) +CONTEXT_TIERS: tuple[str, ...] = ("default", "long_context") + def _copilot_executable() -> str: """Return the executable name for Copilot CLI on this platform. @@ -351,8 +360,9 @@ def _dispatch_setting_args( """Gate per-step dispatch settings and return their Copilot CLI flags. *via* names the method that received the settings, as for - :func:`check_dispatch_settings`. Values pass through unchanged; the - CLI rejects levels or tiers it does not recognize. + :func:`check_dispatch_settings`. The gate calls + :meth:`validate_dispatch_setting`, so only values the CLI accepts + reach it. """ settings = requested_dispatch_settings( reasoning_effort=reasoning_effort, context_size=context_size @@ -367,6 +377,27 @@ def _dispatch_setting_args( args.extend(["--context", context_size]) return args + def validate_dispatch_setting(self, name: str, value: str) -> None: + """Reject values that Copilot CLI does not accept. + + Copilot CLI repeats a rejected value in its error, and the value can be + captured agent output, so Spec Kit rejects it first without repeating it. + """ + if name == "reasoning_effort" and value not in REASONING_EFFORT_LEVELS: + raise ValueError( + "GitHub Copilot 'reasoning_effort' must be one of " + f"{', '.join(REASONING_EFFORT_LEVELS)}. To pass a newer level, " + 'set SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--reasoning-effort ' + "\" instead of 'reasoning_effort'." + ) + if name == "context_size" and value not in CONTEXT_TIERS: + raise ValueError( + "GitHub Copilot 'context_size' must be one of " + f"{', '.join(CONTEXT_TIERS)}. To pass a newer tier, set " + 'SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--context " ' + "instead of 'context_size'." + ) + def build_exec_args( self, prompt: str, diff --git a/tests/integrations/test_integration_copilot.py b/tests/integrations/test_integration_copilot.py index 8563150bfa..41d9dd674c 100644 --- a/tests/integrations/test_integration_copilot.py +++ b/tests/integrations/test_integration_copilot.py @@ -12,15 +12,21 @@ import yaml from specify_cli.integrations import get_integration +from specify_cli.integrations._capture import CapturedProcess from specify_cli.integrations.base import ( DispatchSettingsError, SkillsIntegration, check_dispatch_settings, effective_dispatch_settings, ) -from specify_cli.integrations.copilot import CopilotIntegration +from specify_cli.integrations.copilot import ( + CONTEXT_TIERS, + REASONING_EFFORT_LEVELS, + CopilotIntegration, +) from specify_cli.integrations.manifest import IntegrationManifest -from specify_cli.workflows.base import StepContext, StepStatus +from specify_cli.workflows.base import RunStatus, StepContext, StepStatus +from specify_cli.workflows.engine import WorkflowDefinition, WorkflowEngine from specify_cli.workflows.step.command import CommandStep from specify_cli.workflows.step.prompt import PromptStep @@ -1296,6 +1302,14 @@ def build_exec_args( _COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) _PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) +# Captured agent output that passes the host's value pattern, so only +# Copilot's own value check rejects it. +_SECRET_OUTPUT = "FAKEsecret0123456789" +_ACCEPTED_VALUES = pytest.mark.parametrize( + ("setting", "flag", "value"), + [("reasoning_effort", "--reasoning-effort", v) for v in REASONING_EFFORT_LEVELS] + + [("context_size", "--context", v) for v in CONTEXT_TIERS], +) class TestCopilotDispatchSettings: @@ -1423,11 +1437,63 @@ def test_unset_settings_leave_argv_unchanged( expected = [*dispatch_head, "--model", "gpt-5", "--output-format", "json"] assert runs == [expected, expected] - def test_values_pass_through_for_the_cli_to_validate(self): - args = self._copilot().build_exec_args( - "do it", reasoning_effort="bogus", context_size="huge" + def test_accepted_values_match_copilot_cli(self): + assert REASONING_EFFORT_LEVELS == ( + "none", "minimal", "low", "medium", "high", "xhigh", "max" + ) + assert CONTEXT_TIERS == ("default", "long_context") + + @_ACCEPTED_VALUES + def test_every_accepted_value_reaches_the_cli(self, runs, setting, flag, value): + copilot = self._copilot() + assert copilot.build_exec_args("do it", **{setting: value}) == [ + "copilot", "-p", "do it", flag, value, "--output-format", "json" + ] + copilot.dispatch_command( + "speckit.plan", "body", stream=False, **{setting: value} + ) + assert runs == [ + ["copilot", "-p", "/speckit-plan body", flag, value, + "--output-format", "json"] + ] + + @pytest.mark.usefixtures("no_runs") + @pytest.mark.parametrize( + ("setting", "value", "accepted", "hatch"), + [ + ("reasoning_effort", "bogus", REASONING_EFFORT_LEVELS, "--reasoning-effort"), + ("reasoning_effort", "HIGH", REASONING_EFFORT_LEVELS, "--reasoning-effort"), + ("reasoning_effort", _SECRET_OUTPUT, REASONING_EFFORT_LEVELS, "--reasoning-effort"), + ("context_size", "huge", CONTEXT_TIERS, "--context"), + ("context_size", "200000", CONTEXT_TIERS, "--context"), + ("context_size", _SECRET_OUTPUT, CONTEXT_TIERS, "--context"), + ], + ) + def test_unknown_values_fail_before_the_cli_starts( + self, setting, value, accepted, hatch + ): + copilot = self._copilot() + with pytest.raises(DispatchSettingsError) as built: + copilot.build_exec_args("do it", **{setting: value}) + with pytest.raises(DispatchSettingsError) as dispatched: + copilot.dispatch_command("speckit.plan", stream=False, **{setting: value}) + for excinfo in (built, dispatched): + message = str(excinfo.value) + assert excinfo.value.setting == setting + assert value not in message + assert f"must be one of {', '.join(accepted)}." in message + assert ( + f'SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="{hatch} "' in message + ) + + def test_extra_args_escape_hatch_passes_newer_values(self, monkeypatch): + monkeypatch.setenv( + "SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS", "--reasoning-effort ultra" ) - assert args[3:7] == ["--reasoning-effort", "bogus", "--context", "huge"] + assert self._copilot().build_exec_args("do it") == [ + "copilot", "-p", "do it", "--reasoning-effort", "ultra", + "--output-format", "json", + ] @pytest.mark.usefixtures("no_runs") @pytest.mark.parametrize( @@ -1504,3 +1570,68 @@ def test_workflow_steps_dispatch_settings( assert args[1:] == [ "-p", prompt, "--reasoning-effort", "high", "--context", "long_context" ] + + @pytest.mark.parametrize("setting", ["reasoning_effort", "context_size"]) + @pytest.mark.parametrize( + ("label", "step"), + [("Command", _COMMAND_STEP[1]), ("Prompt", _PROMPT_STEP[1])], + ids=["command", "prompt"], + ) + def test_captured_secret_stays_out_of_the_cli_error_and_log( + self, monkeypatch, tmp_path, label, step, setting + ): + """A streaming step that reads captured output into a setting fails + before Copilot CLI could repeat the value.""" + captured, streamed = [], [] + + def run_captured(args, **kwargs): + captured.append(list(args)) + return CapturedProcess(exit_code=0, stdout=_SECRET_OUTPUT, stderr="") + + def run(args, **kwargs): + streamed.append(list(args)) + return subprocess.CompletedProcess(args, 1, stdout="", stderr="") + + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + monkeypatch.setattr( + "specify_cli.workflows.step.prompt.run_captured", run_captured + ) + monkeypatch.setattr(subprocess, "run", run) + (tmp_path / ".specify" / "workflows").mkdir(parents=True) + definition = WorkflowDefinition( + { + "schema_version": "1.0", + "workflow": { + "id": "leak", + "name": "leak", + "version": "1.0.0", + "integration": "copilot", + }, + "steps": [ + { + "id": "read", + "type": "prompt", + "prompt": "Print the token", + "capture": True, + }, + {"id": "ask", **step, setting: "{{ steps.read.output.stdout }}"}, + ], + } + ) + + state = WorkflowEngine(tmp_path).execute(definition) + + assert state.step_results["read"]["output"]["stdout"] == _SECRET_OUTPUT + assert len(captured) == 1 + assert streamed == [] + assert state.status is RunStatus.FAILED + error = state.step_results["ask"]["error"] + assert error.startswith( + f"{label} step 'ask': GitHub Copilot '{setting}' must be one of " + ) + assert error.endswith(" (set by step)") + assert _SECRET_OUTPUT not in error + log = ( + tmp_path / ".specify" / "workflows" / "runs" / state.run_id / "log.jsonl" + ).read_text(encoding="utf-8") + assert _SECRET_OUTPUT not in log From 64329bbce9e57b936ea54191b282552f2846606f Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Fri, 6 Nov 2026 15:24:42 -0600 Subject: [PATCH 7/8] docs(codex): keep captured output out of reasoning effort Codex displays the reasoning effort it is given and accepts any value a model advertises, so Spec Kit cannot check it against a fixed list. The guide now advises setting it from a literal or a workflow input rather than from captured output. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/integrations/codex.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/integrations/codex.md b/docs/integrations/codex.md index 967f11db2b..fcd89d5520 100644 --- a/docs/integrations/codex.md +++ b/docs/integrations/codex.md @@ -49,8 +49,10 @@ Codex dispatch passes them as configuration overrides after `--model`: | `context_size` | `-c model_context_window=` | A token count: a positive whole number without leading zeros, at most `9223372036854775807` | Spec Kit passes `reasoning_effort` through unchanged; Codex CLI 0.165.0 -forwards it to the model without checking it. A `context_size` that is not a -valid token count fails the step before Codex starts. +forwards it to the model without checking it. Codex displays the reasoning +effort it is given, so set it from a literal or a workflow input, not from +captured output. A `context_size` that is not a valid token count fails the +step before Codex starts. `context_size` overrides the context window that Codex assumes for the model. Codex clamps the value to the model's maximum, and a value below the model's From d5fcab0fe5bb2fdf70b93aa60226506b9b783d3c Mon Sep 17 00:00:00 2001 From: Sukanth Gunda Date: Mon, 9 Nov 2026 07:55:41 -0600 Subject: [PATCH 8/8] docs(workflows): run the context size example on Codex The shared example sets context_size: "200000" on a step that inherits the workflow's integration. Of the built-in integrations, only Codex takes a token count, so the step failed when the workflow ran on GitHub Copilot or Claude Code. The example now dispatches that step to Codex. Part of #4916. Assisted-by: GitHub Copilot (model: Claude Opus 5.5, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/workflow-steps/types.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/workflow-steps/types.md b/docs/workflow-steps/types.md index 32d5434671..6e96eec5cd 100644 --- a/docs/workflow-steps/types.md +++ b/docs/workflow-steps/types.md @@ -223,6 +223,7 @@ inputs: steps: - id: plan command: speckit.plan + integration: codex # Codex takes context_size as a token count reasoning_effort: "high" # Overrides the workflow default context_size: "200000" # Quoted: values are strings