diff --git a/docs/integrations/claude.md b/docs/integrations/claude.md index 5f0fa5eacc..3bb067f1fb 100644 --- a/docs/integrations/claude.md +++ b/docs/integrations/claude.md @@ -69,3 +69,18 @@ See [runtime events](cli.md#runtime-events). Workflow steps use `claude -p `, with `--model` when configured and `--output-format json` when structured output is requested. This adapter does not add a permission-bypass flag itself. + +## Workflow Dispatch Settings + +Command and prompt steps can set +[`reasoning_effort`](../workflow-steps/types.md#reasoning-effort-and-context-size), +which Claude Code dispatch passes as `--effort `. Claude Code 2.1.318 +accepts `low`, `medium`, `high`, `xhigh`, and `max`. It ignores any other +value with only a warning, so Spec Kit rejects other values before Claude Code +starts. To pass a level that a newer Claude Code release adds, set +`SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort "` instead of +`reasoning_effort`; the environment variable applies to every Claude Code +invocation, not just one step. + +Spec Kit does not map `context_size` for Claude Code, so a step that sets it +fails. Choose a long-context model with the step's `model` field instead. diff --git a/docs/integrations/cli.md b/docs/integrations/cli.md index d67252af5d..91ccf0f07b 100644 --- a/docs/integrations/cli.md +++ b/docs/integrations/cli.md @@ -490,7 +490,9 @@ The [`reasoning_effort` and `context_size`](../workflow-steps/types.md#reasoning settings of `command` and `prompt` steps instead follow one shared contract. Spec Kit checks each value and passes it only to an integration that declares and translates that setting; with any other integration, the step fails before -the CLI runs. Adapter authors opt in as described in the +the CLI runs. GitHub Copilot and Codex CLI translate both settings, and Claude +Code translates `reasoning_effort`; the agent guides describe each mapping. +Adapter authors opt in as described in the [integration API reference](api.md#per-step-dispatch-settings). ## FAQ diff --git a/docs/integrations/codex.md b/docs/integrations/codex.md index d37b6901f5..fcd89d5520 100644 --- a/docs/integrations/codex.md +++ b/docs/integrations/codex.md @@ -36,3 +36,24 @@ See [runtime events](cli.md#runtime-events). Workflow steps use `codex exec `, with `--model` when configured and `--json` when structured output is requested. Codex dev-mode registration writes skill files rather than symlinking them. + +## Workflow Dispatch Settings + +Command and prompt steps can set +[`reasoning_effort` and `context_size`](../workflow-steps/types.md#reasoning-effort-and-context-size). +Codex dispatch passes them as configuration overrides after `--model`: + +| Step setting | Codex CLI argument | Values | +|---|---|---| +| `reasoning_effort` | `-c model_reasoning_effort=""` | A level the selected model supports, such as `low`, `medium`, `high`, or `xhigh` | +| `context_size` | `-c model_context_window=` | A token count: a positive whole number without leading zeros, at most `9223372036854775807` | + +Spec Kit passes `reasoning_effort` through unchanged; Codex CLI 0.165.0 +forwards it to the model without checking it. Codex displays the reasoning +effort it is given, so set it from a literal or a workflow input, not from +captured output. A `context_size` that is not a valid token count fails the +step before Codex starts. + +`context_size` overrides the context window that Codex assumes for the model. +Codex clamps the value to the model's maximum, and a value below the model's +default shrinks the usable context. diff --git a/docs/integrations/copilot.md b/docs/integrations/copilot.md index b0a5b57686..12075e9b82 100644 --- a/docs/integrations/copilot.md +++ b/docs/integrations/copilot.md @@ -116,3 +116,23 @@ without interactive approval. Set `SPECKIT_COPILOT_ALLOW_ALL_TOOLS=0` to stop Spec Kit from adding this flag. The deprecated `SPECKIT_ALLOW_ALL_TOOLS` is accepted only as a fallback. A configured `--model` and requested `--output-format json` are forwarded. + +## Workflow Dispatch Settings + +Command and prompt steps can set +[`reasoning_effort` and `context_size`](../workflow-steps/types.md#reasoning-effort-and-context-size). +Copilot CLI dispatch passes them as native flags after `--model`, in both the +skills and commands layouts: + +| Step setting | Copilot CLI flag | Values in Copilot CLI 1.0.106 | +|---|---|---| +| `reasoning_effort` | `--reasoning-effort ` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | +| `context_size` | `--context ` | `default`, `long_context` | + +Spec Kit accepts only these values. Any other value fails the step before +Copilot CLI starts, because Copilot CLI repeats a rejected value in its error +and the value can come from captured output. To pass a value that a newer +Copilot CLI release adds, set +`SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--reasoning-effort "` or +`SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--context "` instead of the +step setting. diff --git a/docs/workflow-steps/types.md b/docs/workflow-steps/types.md index 9a22906ce0..6e96eec5cd 100644 --- a/docs/workflow-steps/types.md +++ b/docs/workflow-steps/types.md @@ -32,6 +32,7 @@ Invoke an installed Spec Kit command by name via the integration CLI: args: "{{ inputs.spec }}" integration: claude # Optional: override workflow default model: "claude-sonnet-4-20250514" # Optional: override model + reasoning_effort: "high" # Optional: requires integration support ``` For positional arguments and named runtime options, see @@ -222,6 +223,7 @@ inputs: steps: - id: plan command: speckit.plan + integration: codex # Codex takes context_size as a token count reasoning_effort: "high" # Overrides the workflow default context_size: "200000" # Quoted: values are strings @@ -254,8 +256,17 @@ integration's CLI, and an integration can reject a value before the CLI runs. for it. Otherwise the step fails before the CLI runs, with an error that names the setting, its source (the step or the workflow default), and any integrations that support it. Workflow defaults are not exempt: a default fails -each inheriting step whose integration does not support it. No built-in -integration supports these settings yet; see +each inheriting step whose integration does not support it. These built-in +integrations support the settings: + +| Integration | `reasoning_effort` | `context_size` | +| --- | --- | --- | +| [GitHub Copilot](../integrations/copilot.md#workflow-dispatch-settings) | Yes | Yes | +| [Codex CLI](../integrations/codex.md#workflow-dispatch-settings) | Yes | Yes | +| [Claude Code](../integrations/claude.md#workflow-dispatch-settings) | Yes | No | + +Other built-in integrations support neither setting. Each integration's guide +describes its CLI arguments and accepted values; see [Workflow CLI dispatch](../integrations/cli.md#workflow-cli-dispatch) for how integrations opt in. diff --git a/src/specify_cli/integrations/claude/__init__.py b/src/specify_cli/integrations/claude/__init__.py index e2de919b8c..641480fe82 100644 --- a/src/specify_cli/integrations/claude/__init__.py +++ b/src/specify_cli/integrations/claude/__init__.py @@ -2,13 +2,24 @@ from __future__ import annotations +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any -from ..base import IntegrationOption, SkillsIntegration +from ..base import ( + IntegrationOption, + SkillsIntegration, + check_dispatch_settings, + requested_dispatch_settings, +) from ..manifest import IntegrationManifest from ..._utils import dump_frontmatter +# ``--effort`` levels accepted by Claude Code 2.1.318. The CLI only warns about +# any other value and runs with its default effort, so per-step +# ``reasoning_effort`` is limited to these. +EFFORT_LEVELS: tuple[str, ...] = ("low", "medium", "high", "xhigh", "max") + # Mapping of command template stem → argument-hint text shown inline # when a user invokes the slash command in Claude Code. ARGUMENT_HINTS: dict[str, str] = { @@ -65,6 +76,7 @@ class ClaudeIntegration(SkillsIntegration): } events_config_file = ".claude/settings.json" events_format = "json-nested" + supported_dispatch_settings = frozenset({"reasoning_effort"}) # Set by ``setup()`` from ``--no-model-invocation``; applies to core # command skills only, since mandatory extension hooks rely on the model @@ -91,6 +103,59 @@ def options(cls) -> list[IntegrationOption]: ) return opts + def build_exec_args( + self, + prompt: str, + *, + model: str | None = None, + output_json: bool = True, + integration_args: Sequence[str] | None = None, + integration_options: Mapping[str, Any] | None = None, + project_root: Path | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, + ) -> list[str] | None: + """Build ``claude -p`` arguments, adding ``--effort`` when requested.""" + # super() validates the runtime config, once, before the gate runs, so + # its errors come first, as in dispatch_command(). + args = super().build_exec_args( + prompt, + model=model, + output_json=output_json, + integration_args=integration_args, + integration_options=integration_options, + project_root=project_root, + ) + settings = requested_dispatch_settings( + reasoning_effort=reasoning_effort, context_size=context_size + ) + if settings: + check_dispatch_settings(self, settings, via="build_exec_args") + if args is not None and reasoning_effort is not None: + args.extend(["--effort", reasoning_effort]) + return args + + def validate_dispatch_setting(self, name: str, value: str) -> None: + """Reject effort levels that Claude Code would silently ignore. + + The value can be captured agent output, so the error does not repeat it. + """ + if name == "reasoning_effort" and value not in EFFORT_LEVELS: + raise ValueError( + "Claude Code 'reasoning_effort' must be one of " + f"{', '.join(EFFORT_LEVELS)}. To pass a newer level, set " + 'SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort " ' + "instead of 'reasoning_effort'." + ) + + def dispatch_setting_hint(self, setting: str) -> str | None: + if setting == "context_size": + return ( + "Choose a long-context model with the step's 'model' field " + "instead." + ) + return None + def setup( self, project_root: Path, diff --git a/src/specify_cli/integrations/codex/__init__.py b/src/specify_cli/integrations/codex/__init__.py index d88628b3ff..d346ff3a59 100644 --- a/src/specify_cli/integrations/codex/__init__.py +++ b/src/specify_cli/integrations/codex/__init__.py @@ -5,12 +5,22 @@ """ from __future__ import annotations +import re from pathlib import Path from collections.abc import Mapping, Sequence from typing import Any -from ..base import IntegrationOption, SkillsIntegration +from ..base import ( + IntegrationOption, + SkillsIntegration, + check_dispatch_settings, + requested_dispatch_settings, +) + +# ``model_context_window`` is a TOML integer that Codex reads as an i64. +_CONTEXT_WINDOW = re.compile(r"[1-9][0-9]*") +_MAX_CONTEXT_WINDOW = 2**63 - 1 class CodexIntegration(SkillsIntegration): @@ -32,6 +42,7 @@ class CodexIntegration(SkillsIntegration): } dev_no_symlink = True multi_install_safe = True + supported_dispatch_settings = frozenset({"reasoning_effort", "context_size"}) CANONICAL_TO_NATIVE = { "session_start": "SessionStart", @@ -53,19 +64,48 @@ def build_exec_args( integration_args: Sequence[str] | None = None, integration_options: Mapping[str, Any] | None = None, project_root: Path | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, ) -> list[str] | None: # Codex uses ``codex exec "prompt"`` for non-interactive mode. # Resolve argv[0] via the shared executable resolver so operators can # override the binary with SPECKIT_INTEGRATION_CODEX_EXECUTABLE. self.validate_runtime_config(integration_args, integration_options) + settings = requested_dispatch_settings( + reasoning_effort=reasoning_effort, context_size=context_size + ) + if settings: + check_dispatch_settings(self, settings, via="build_exec_args") args: list[str] = [self._resolve_executable(), "exec", prompt] self._apply_extra_args_env_var(args) if model: args.extend(["--model", model]) + # ``-c`` values are parsed as TOML. The checked value cannot contain + # quotes, backslashes, or control characters, so it forms a valid + # basic string as-is. + if reasoning_effort is not None: + args.extend(["-c", f'model_reasoning_effort="{reasoning_effort}"']) + if context_size is not None: + args.extend(["-c", f"model_context_window={context_size}"]) if output_json: args.append("--json") return args + def validate_dispatch_setting(self, name: str, value: str) -> None: + """Require ``context_size`` to be a token count Codex can parse. + + The value can be captured agent output, so the error does not repeat it. + """ + if name == "context_size" and ( + _CONTEXT_WINDOW.fullmatch(value) is None + or int(value) > _MAX_CONTEXT_WINDOW + ): + raise ValueError( + "Codex 'context_size' must be a positive integer token count " + "(model_context_window) without leading zeros, at most " + f"{_MAX_CONTEXT_WINDOW}." + ) + @classmethod def options(cls) -> list[IntegrationOption]: opts = super().options() diff --git a/src/specify_cli/integrations/copilot/__init__.py b/src/specify_cli/integrations/copilot/__init__.py index 6f373613bb..205c25407a 100644 --- a/src/specify_cli/integrations/copilot/__init__.py +++ b/src/specify_cli/integrations/copilot/__init__.py @@ -30,7 +30,13 @@ import typer from .._capture import run_captured -from ..base import IntegrationBase, IntegrationOption, SkillsIntegration +from ..base import ( + IntegrationBase, + IntegrationOption, + SkillsIntegration, + check_dispatch_settings, + requested_dispatch_settings, +) from ..manifest import IntegrationManifest _COPILOT_CORE_COMMANDS = { @@ -45,6 +51,15 @@ "tasks", } +# ``--reasoning-effort`` levels and ``--context`` tiers accepted by GitHub +# Copilot CLI 1.0.106. The CLI rejects any other value with an error that +# repeats it, and a value can be captured agent output, so per-step +# ``reasoning_effort`` and ``context_size`` are limited to these. +REASONING_EFFORT_LEVELS: tuple[str, ...] = ( + "none", "minimal", "low", "medium", "high", "xhigh", "max" +) +CONTEXT_TIERS: tuple[str, ...] = ("default", "long_context") + def _copilot_executable() -> str: """Return the executable name for Copilot CLI on this platform. @@ -152,6 +167,9 @@ class CopilotIntegration(IntegrationBase): "extension": ".agent.md", } invoke_separator = "-" + # Translated by both receiving methods defined below: build_exec_args() + # for prompt steps and dispatch_command() for command steps. + supported_dispatch_settings = frozenset({"reasoning_effort", "context_size"}) CANONICAL_TO_NATIVE = { "session_start": "sessionStart", @@ -332,6 +350,54 @@ def _resolve_executable(self) -> str: override = os.environ.get(env_name, "").strip() return override if override else _copilot_executable() + def _dispatch_setting_args( + self, + *, + via: str, + reasoning_effort: str | None, + context_size: str | None, + ) -> list[str]: + """Gate per-step dispatch settings and return their Copilot CLI flags. + + *via* names the method that received the settings, as for + :func:`check_dispatch_settings`. The gate calls + :meth:`validate_dispatch_setting`, so only values the CLI accepts + reach it. + """ + settings = requested_dispatch_settings( + reasoning_effort=reasoning_effort, context_size=context_size + ) + if not settings: + return [] + check_dispatch_settings(self, settings, via=via) + args: list[str] = [] + if reasoning_effort is not None: + args.extend(["--reasoning-effort", reasoning_effort]) + if context_size is not None: + args.extend(["--context", context_size]) + return args + + def validate_dispatch_setting(self, name: str, value: str) -> None: + """Reject values that Copilot CLI does not accept. + + Copilot CLI repeats a rejected value in its error, and the value can be + captured agent output, so Spec Kit rejects it first without repeating it. + """ + if name == "reasoning_effort" and value not in REASONING_EFFORT_LEVELS: + raise ValueError( + "GitHub Copilot 'reasoning_effort' must be one of " + f"{', '.join(REASONING_EFFORT_LEVELS)}. To pass a newer level, " + 'set SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--reasoning-effort ' + "\" instead of 'reasoning_effort'." + ) + if name == "context_size" and value not in CONTEXT_TIERS: + raise ValueError( + "GitHub Copilot 'context_size' must be one of " + f"{', '.join(CONTEXT_TIERS)}. To pass a newer tier, set " + 'SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="--context " ' + "instead of 'context_size'." + ) + def build_exec_args( self, prompt: str, @@ -341,8 +407,15 @@ def build_exec_args( integration_args: Sequence[str] | None = None, integration_options: Mapping[str, Any] | None = None, project_root: Path | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, ) -> list[str] | None: self.validate_runtime_config(integration_args, integration_options) + setting_args = self._dispatch_setting_args( + via="build_exec_args", + reasoning_effort=reasoning_effort, + context_size=context_size, + ) # GitHub Copilot CLI uses ``copilot -p "prompt"`` for # non-interactive mode. --yolo enables all permissions # (tools, paths, and URLs) so the agent can perform file @@ -356,6 +429,7 @@ def build_exec_args( args.append("--yolo") if model: args.extend(["--model", model]) + args.extend(setting_args) if output_json: args.extend(["--output-format", "json"]) return args @@ -389,6 +463,8 @@ def dispatch_command( integration_options: Mapping[str, Any] | None = None, output_json: bool | None = None, capture_max_bytes: int | None = None, + reasoning_effort: str | None = None, + context_size: str | None = None, ) -> dict[str, Any]: """Dispatch via ``--agent speckit.`` instead of slash-commands. @@ -407,6 +483,11 @@ def dispatch_command( output_json = not stream self.validate_runtime_config(integration_args, integration_options) + setting_args = self._dispatch_setting_args( + via="dispatch_command", + reasoning_effort=reasoning_effort, + context_size=context_size, + ) stem = command_name if stem.startswith("speckit."): @@ -438,6 +519,7 @@ def dispatch_command( cli_args.append("--yolo") if model: cli_args.extend(["--model", model]) + cli_args.extend(setting_args) if output_json: cli_args.extend(["--output-format", "json"]) diff --git a/tests/integrations/test_integration_claude.py b/tests/integrations/test_integration_claude.py index f275737075..5cc056d1e5 100644 --- a/tests/integrations/test_integration_claude.py +++ b/tests/integrations/test_integration_claude.py @@ -2,6 +2,8 @@ import json import os +import shutil +import subprocess from pathlib import Path from unittest.mock import patch @@ -9,9 +11,22 @@ import yaml from specify_cli.integrations import INTEGRATION_REGISTRY, get_integration -from specify_cli.integrations.base import IntegrationBase, SkillsIntegration -from specify_cli.integrations.claude import ARGUMENT_HINTS, FORK_CONTEXT_COMMANDS +from specify_cli.integrations.base import ( + DispatchSettingsError, + IntegrationBase, + SkillsIntegration, + check_dispatch_settings, + effective_dispatch_settings, +) +from specify_cli.integrations.claude import ( + ARGUMENT_HINTS, + EFFORT_LEVELS, + FORK_CONTEXT_COMMANDS, +) from specify_cli.integrations.manifest import IntegrationManifest +from specify_cli.workflows.base import StepContext, StepStatus +from specify_cli.workflows.step.command import CommandStep +from specify_cli.workflows.step.prompt import PromptStep REPO_ROOT = Path(__file__).resolve().parents[2] @@ -1248,3 +1263,231 @@ def test_install_shared_infra_handles_directory_at_template_destination( assert real_template in text, ( "directory-at-template-dst path must surface in the skipped warning" ) + + +_COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) +_PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) +# Captured agent output that passes the host's value pattern, so only Claude's +# own rule rejects it. +_SECRET_OUTPUT = "FAKEsecret0123456789" + + +class TestClaudeDispatchSettings: + """Per-step ``reasoning_effort`` maps to ``--effort`` (#4916).""" + + @pytest.fixture + def runs(self, monkeypatch): + calls = [] + + def fake_run(args, **kwargs): + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + @pytest.mark.parametrize("via", ["dispatch_command", "build_exec_args"]) + def test_build_exec_args_owns_reasoning_effort_only(self, via): + from specify_cli.integrations.claude import ClaudeIntegration + + assert "build_exec_args" in vars(ClaudeIntegration) + assert effective_dispatch_settings( + ClaudeIntegration(), via=via + ) == frozenset({"reasoning_effort"}) + + def test_effort_levels_match_claude_code(self): + assert EFFORT_LEVELS == ("low", "medium", "high", "xhigh", "max") + + @pytest.mark.parametrize("level", ["low", "medium", "high", "xhigh", "max"]) + def test_every_effort_level_becomes_the_effort_flag(self, level): + claude = get_integration("claude") + check_dispatch_settings( + claude, {"reasoning_effort": level}, via="build_exec_args" + ) + assert claude.build_exec_args( + "do it", model="sonnet", reasoning_effort=level + ) == [ + "claude", "-p", "do it", + "--model", "sonnet", + "--output-format", "json", + "--effort", level, + ] + + def test_dispatch_command_passes_effort(self, runs): + get_integration("claude").dispatch_command( + "speckit.plan", "body", stream=False, reasoning_effort="max" + ) + (args,) = runs + assert args[1:] == [ + "-p", "/speckit-plan body", "--output-format", "json", "--effort", "max" + ] + + @pytest.mark.parametrize("level", ["bogus", "none", "minimal", "HIGH"]) + def test_unknown_effort_fails_before_the_cli_starts(self, runs, level): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("claude").dispatch_command( + "speckit.plan", stream=False, reasoning_effort=level + ) + assert runs == [] + assert excinfo.value.setting == "reasoning_effort" + message = str(excinfo.value) + assert "must be one of low, medium, high, xhigh, max" in message + assert 'SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort "' in message + + def test_direct_build_rejects_unknown_effort(self): + with pytest.raises( + DispatchSettingsError, match="SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS" + ) as excinfo: + get_integration("claude").build_exec_args( + "do it", reasoning_effort="bogus" + ) + assert excinfo.value.setting == "reasoning_effort" + + def test_rejected_effort_is_not_echoed(self): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("claude").build_exec_args( + "do it", reasoning_effort=_SECRET_OUTPUT + ) + message = str(excinfo.value) + assert _SECRET_OUTPUT not in message + assert "must be one of low, medium, high, xhigh, max." in message + + def test_context_size_is_rejected_with_a_model_hint(self, runs): + claude = get_integration("claude") + with pytest.raises(DispatchSettingsError) as excinfo: + claude.dispatch_command( + "speckit.plan", stream=False, context_size="long_context" + ) + assert runs == [] + assert excinfo.value.setting == "context_size" + message = str(excinfo.value) + assert "does not support per-step 'context_size'" in message + assert ( + "Choose a long-context model with the step's 'model' field instead." + in message + ) + assert claude.dispatch_setting_hint("reasoning_effort") is None + + def test_context_size_is_attributed_when_both_settings_are_set(self): + with pytest.raises(DispatchSettingsError, match="'model' field") as excinfo: + get_integration("claude").build_exec_args( + "do it", reasoning_effort="high", context_size="long_context" + ) + assert excinfo.value.setting == "context_size" + + def test_unset_settings_leave_argv_unchanged(self): + claude = get_integration("claude") + baseline = [ + "claude", "-p", "do it", "--model", "sonnet", "--output-format", "json" + ] + assert claude.build_exec_args("do it", model="sonnet") == baseline + assert claude.build_exec_args( + "do it", model="sonnet", reasoning_effort=None, context_size=None + ) == baseline + + def test_runtime_config_is_validated_once(self, monkeypatch): + from specify_cli.integrations.claude import ClaudeIntegration + + claude = ClaudeIntegration() + calls = [] + monkeypatch.setattr( + claude, + "validate_runtime_config", + lambda *args: calls.append(args), + ) + + claude.build_exec_args("do it", integration_args=["a"]) + assert calls == [(["a"], None)] + + calls.clear() + claude.build_exec_args( + "do it", integration_args=["a"], reasoning_effort="high" + ) + assert calls == [(["a"], None)] + + def test_runtime_config_errors_precede_setting_errors(self): + with pytest.raises(ValueError, match="'integration_args'"): + get_integration("claude").build_exec_args( + "do it", integration_args=["x"], reasoning_effort="bogus" + ) + + def test_extra_args_escape_hatch_passes_newer_levels(self, monkeypatch): + monkeypatch.setenv("SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS", "--effort ultra") + assert get_integration("claude").build_exec_args("do it") == [ + "claude", "-p", "do it", "--effort", "ultra", "--output-format", "json" + ] + + @staticmethod + def _execute(monkeypatch, tmp_path, step, config, *, steps=None, **settings): + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + return step().execute( + {"id": "ask", **config, **settings}, + StepContext( + default_integration="claude", + project_root=str(tmp_path), + steps=steps or {}, + ), + ) + + @pytest.mark.parametrize( + ("step", "config", "prompt"), + [(*_COMMAND_STEP, "/speckit-plan body"), (*_PROMPT_STEP, "Summarise")], + ids=["command", "prompt"], + ) + def test_workflow_steps_dispatch_effort( + self, monkeypatch, tmp_path, runs, step, config, prompt + ): + result = self._execute( + monkeypatch, tmp_path, step, config, reasoning_effort="high" + ) + assert result.status == StepStatus.COMPLETED, result.error + assert result.output["reasoning_effort"] == "high" + assert result.output["context_size"] is None + (args,) = runs + assert args[1:] == ["-p", prompt, "--effort", "high"] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ( + {"reasoning_effort": "bogus"}, + 'SPECKIT_INTEGRATION_CLAUDE_EXTRA_ARGS="--effort "', + ), + ( + {"context_size": "long_context"}, + "Choose a long-context model with the step's 'model' field", + ), + ], + ids=["unknown-effort", "context-size"], + ) + def test_workflow_steps_fail_before_the_cli_starts( + self, monkeypatch, tmp_path, runs, step, config, settings, expected + ): + result = self._execute(monkeypatch, tmp_path, step, config, **settings) + assert result.status == StepStatus.FAILED + assert result.output["dispatched"] is False + assert expected in result.error + assert runs == [] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + def test_workflow_steps_do_not_echo_captured_output( + self, monkeypatch, tmp_path, runs, step, config + ): + result = self._execute( + monkeypatch, + tmp_path, + step, + config, + steps={"probe": {"output": {"stdout": _SECRET_OUTPUT}}}, + reasoning_effort="{{ steps.probe.output.stdout }}", + ) + assert result.status == StepStatus.FAILED + assert _SECRET_OUTPUT not in result.error + assert "must be one of low, medium, high, xhigh, max." in result.error + assert result.error.endswith(" (set by step)") + assert runs == [] diff --git a/tests/integrations/test_integration_codex.py b/tests/integrations/test_integration_codex.py index 9a4dc5442b..abf4cb289c 100644 --- a/tests/integrations/test_integration_codex.py +++ b/tests/integrations/test_integration_codex.py @@ -1,7 +1,20 @@ """Tests for CodexIntegration.""" +import shutil +import subprocess +import tomllib + +import pytest + from specify_cli.integrations import get_integration +from specify_cli.integrations.base import ( + DispatchSettingsError, + effective_dispatch_settings, +) from specify_cli.integrations.manifest import IntegrationManifest +from specify_cli.workflows.base import StepContext, StepStatus +from specify_cli.workflows.step.command import CommandStep +from specify_cli.workflows.step.prompt import PromptStep from .test_integration_base_skills import SkillsIntegrationTests @@ -191,3 +204,199 @@ def test_hook_note_when_instruction_is_final_line_without_newline(self): assert "For each executable hook" not in lines[note_line_idx], ( "Note and instruction must not be on the same line" ) + + +_COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) +_PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) +# Captured agent output that passes the host's value pattern, so only Codex's +# own rule rejects it. +_SECRET_OUTPUT = "FAKEsecret0123456789" + + +class TestCodexDispatchSettings: + """Per-step ``reasoning_effort`` and ``context_size`` (#4916).""" + + _CODEX_RULE = "positive integer token count (model_context_window)" + + @pytest.fixture + def runs(self, monkeypatch): + calls = [] + + def fake_run(args, **kwargs): + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + @pytest.mark.parametrize("via", ["dispatch_command", "build_exec_args"]) + def test_build_exec_args_declares_both_settings(self, via): + assert effective_dispatch_settings( + get_integration("codex"), via=via + ) == frozenset({"reasoning_effort", "context_size"}) + + def test_settings_become_config_overrides_after_model(self): + args = get_integration("codex").build_exec_args( + "do it", model="gpt-5", reasoning_effort="high", context_size="200000" + ) + assert args == [ + "codex", "exec", "do it", + "--model", "gpt-5", + "-c", 'model_reasoning_effort="high"', + "-c", "model_context_window=200000", + "--json", + ] + + @pytest.mark.parametrize("value", ["high", "xhigh", "v2.max-ultra_1"]) + def test_reasoning_effort_is_embedded_as_a_toml_string(self, value): + args = get_integration("codex").build_exec_args( + "do it", reasoning_effort=value, output_json=False + ) + assert args[-2] == "-c" + assert tomllib.loads(args[-1]) == {"model_reasoning_effort": value} + + def test_reasoning_effort_values_are_left_to_codex(self): + args = get_integration("codex").build_exec_args( + "do it", reasoning_effort="bogus" + ) + assert args[-3:] == ["-c", 'model_reasoning_effort="bogus"', "--json"] + + @pytest.mark.parametrize("value", ["1", "200000", "9223372036854775807"]) + def test_context_size_accepts_canonical_i64_token_counts(self, value): + args = get_integration("codex").build_exec_args( + "do it", context_size=value, output_json=False + ) + assert args[-2:] == ["-c", f"model_context_window={value}"] + assert tomllib.loads(args[-1]) == {"model_context_window": int(value)} + + @pytest.mark.parametrize( + ("value", "reason"), + [ + ("0", _CODEX_RULE), + ("01", _CODEX_RULE), + ("9223372036854775808", _CODEX_RULE), + ("1e5", _CODEX_RULE), + ("long_context", _CODEX_RULE), + ("-5", "starting with a letter or digit"), + ], + ids=["zero", "leading-zero", "above-i64", "exponent", "tier-name", "negative"], + ) + def test_context_size_rejects_other_values(self, value, reason): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("codex").build_exec_args("do it", context_size=value) + assert excinfo.value.setting == "context_size" + assert reason in str(excinfo.value) + + def test_rejected_context_size_is_not_echoed(self): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("codex").build_exec_args( + "do it", context_size=_SECRET_OUTPUT + ) + message = str(excinfo.value) + assert _SECRET_OUTPUT not in message + assert self._CODEX_RULE in message + + def test_unset_settings_leave_argv_unchanged(self): + codex = get_integration("codex") + baseline = ["codex", "exec", "do it", "--model", "gpt-5", "--json"] + assert codex.build_exec_args("do it", model="gpt-5") == baseline + assert codex.build_exec_args( + "do it", model="gpt-5", reasoning_effort=None, context_size=None + ) == baseline + + def test_dispatch_command_forwards_settings(self, runs): + get_integration("codex").dispatch_command( + "speckit.plan", + "body", + stream=False, + reasoning_effort="low", + context_size="128000", + ) + (args,) = runs + assert args[1:] == [ + "exec", "$speckit-plan body", + "-c", 'model_reasoning_effort="low"', + "-c", "model_context_window=128000", + "--json", + ] + + def test_dispatch_command_rejects_before_the_cli_starts(self, runs): + with pytest.raises(DispatchSettingsError) as excinfo: + get_integration("codex").dispatch_command( + "speckit.plan", stream=False, context_size="long_context" + ) + assert excinfo.value.setting == "context_size" + assert self._CODEX_RULE in str(excinfo.value) + assert runs == [] + + @staticmethod + def _execute(monkeypatch, tmp_path, step, config, *, steps=None, **settings): + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + return step().execute( + {"id": "ask", **config, **settings}, + StepContext( + default_integration="codex", + project_root=str(tmp_path), + steps=steps or {}, + ), + ) + + @pytest.mark.parametrize( + ("step", "config", "prompt"), + [(*_COMMAND_STEP, "$speckit-plan body"), (*_PROMPT_STEP, "Summarise")], + ids=["command", "prompt"], + ) + def test_workflow_steps_dispatch_settings( + self, monkeypatch, tmp_path, runs, step, config, prompt + ): + result = self._execute( + monkeypatch, + tmp_path, + step, + config, + reasoning_effort="high", + context_size="200000", + ) + assert result.status == StepStatus.COMPLETED, result.error + assert result.output["reasoning_effort"] == "high" + assert result.output["context_size"] == "200000" + (args,) = runs + assert args[1:] == [ + "exec", prompt, + "-c", 'model_reasoning_effort="high"', + "-c", "model_context_window=200000", + ] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + def test_workflow_steps_reject_invalid_context_size( + self, monkeypatch, tmp_path, runs, step, config + ): + result = self._execute( + monkeypatch, tmp_path, step, config, context_size="long_context" + ) + assert result.status == StepStatus.FAILED + assert result.output["dispatched"] is False + assert self._CODEX_RULE in result.error + assert runs == [] + + @pytest.mark.parametrize( + ("step", "config"), [_COMMAND_STEP, _PROMPT_STEP], ids=["command", "prompt"] + ) + def test_workflow_steps_do_not_echo_captured_output( + self, monkeypatch, tmp_path, runs, step, config + ): + result = self._execute( + monkeypatch, + tmp_path, + step, + config, + steps={"probe": {"output": {"stdout": _SECRET_OUTPUT}}}, + context_size="{{ steps.probe.output.stdout }}", + ) + assert result.status == StepStatus.FAILED + assert _SECRET_OUTPUT not in result.error + assert self._CODEX_RULE in result.error + assert result.error.endswith(" (set by step)") + assert runs == [] diff --git a/tests/integrations/test_integration_copilot.py b/tests/integrations/test_integration_copilot.py index 796cdc3d97..41d9dd674c 100644 --- a/tests/integrations/test_integration_copilot.py +++ b/tests/integrations/test_integration_copilot.py @@ -2,6 +2,8 @@ import json import os +import shutil +import subprocess from pathlib import Path from unittest.mock import patch @@ -10,8 +12,23 @@ import yaml from specify_cli.integrations import get_integration -from specify_cli.integrations.base import SkillsIntegration +from specify_cli.integrations._capture import CapturedProcess +from specify_cli.integrations.base import ( + DispatchSettingsError, + SkillsIntegration, + check_dispatch_settings, + effective_dispatch_settings, +) +from specify_cli.integrations.copilot import ( + CONTEXT_TIERS, + REASONING_EFFORT_LEVELS, + CopilotIntegration, +) from specify_cli.integrations.manifest import IntegrationManifest +from specify_cli.workflows.base import RunStatus, StepContext, StepStatus +from specify_cli.workflows.engine import WorkflowDefinition, WorkflowEngine +from specify_cli.workflows.step.command import CommandStep +from specify_cli.workflows.step.prompt import PromptStep REPO_ROOT = Path(__file__).resolve().parents[2] @@ -1257,3 +1274,364 @@ def test_extension_hook_skills_stay_model_invocable_after_upgrade(self, tmp_path assert _skill_frontmatter(skills_dir / "speckit-specify" / "SKILL.md")[ "disable-model-invocation" ] is True + + +class _LegacyBuilderCopilot(CopilotIntegration): + """Overrides only ``build_exec_args``, with the signature that predates + per-step dispatch settings.""" + + def build_exec_args( + self, + prompt, + *, + model=None, + output_json=True, + integration_args=None, + integration_options=None, + project_root=None, + ): + return super().build_exec_args( + prompt, + model=model, + output_json=output_json, + integration_args=integration_args, + integration_options=integration_options, + project_root=project_root, + ) + + +_COMMAND_STEP = (CommandStep, {"command": "speckit.plan", "input": {"args": "body"}}) +_PROMPT_STEP = (PromptStep, {"type": "prompt", "prompt": "Summarise"}) +# Captured agent output that passes the host's value pattern, so only +# Copilot's own value check rejects it. +_SECRET_OUTPUT = "FAKEsecret0123456789" +_ACCEPTED_VALUES = pytest.mark.parametrize( + ("setting", "flag", "value"), + [("reasoning_effort", "--reasoning-effort", v) for v in REASONING_EFFORT_LEVELS] + + [("context_size", "--context", v) for v in CONTEXT_TIERS], +) + + +class TestCopilotDispatchSettings: + """Per-step ``reasoning_effort`` and ``context_size`` (#4916).""" + + _LAYOUTS = pytest.mark.parametrize( + ("skills_mode", "dispatch_head"), + [ + (True, ["copilot", "-p", "/speckit-plan body"]), + (False, ["copilot", "-p", "body", "--agent", "speckit.plan"]), + ], + ids=["skills", "commands"], + ) + + @pytest.fixture(autouse=True) + def _deterministic_argv(self, monkeypatch): + monkeypatch.setenv("SPECKIT_INTEGRATION_COPILOT_EXECUTABLE", "copilot") + monkeypatch.setenv("SPECKIT_COPILOT_ALLOW_ALL_TOOLS", "0") + + @pytest.fixture + def runs(self, monkeypatch): + calls = [] + + def fake_run(args, **kwargs): + calls.append(list(args)) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + @pytest.fixture + def no_runs(self, monkeypatch): + def fail(*args, **kwargs): + raise AssertionError("the Copilot CLI must not start") + + monkeypatch.setattr(subprocess, "run", fail) + + @staticmethod + def _copilot(skills_mode=True, cls=CopilotIntegration): + copilot = cls() + copilot._skills_mode = skills_mode + return copilot + + @pytest.mark.parametrize("via", ["dispatch_command", "build_exec_args"]) + def test_both_receiving_methods_declare_both_settings(self, via): + assert effective_dispatch_settings( + get_integration("copilot"), via=via + ) == frozenset({"reasoning_effort", "context_size"}) + + @pytest.mark.parametrize("skills_mode", [True, False], ids=["skills", "commands"]) + def test_build_exec_args_emits_flags_after_model(self, skills_mode): + args = self._copilot(skills_mode).build_exec_args( + "do it", + model="gpt-5", + reasoning_effort="high", + context_size="long_context", + ) + assert args == [ + "copilot", "-p", "do it", + "--model", "gpt-5", + "--reasoning-effort", "high", + "--context", "long_context", + "--output-format", "json", + ] + + @pytest.mark.parametrize( + ("settings", "flags"), + [ + ({"reasoning_effort": "low"}, ["--reasoning-effort", "low"]), + ({"context_size": "default"}, ["--context", "default"]), + ], + ids=["reasoning_effort", "context_size"], + ) + def test_build_exec_args_without_model_emits_only_requested_flags( + self, settings, flags + ): + args = self._copilot().build_exec_args("do it", **settings) + assert args == ["copilot", "-p", "do it", *flags, "--output-format", "json"] + + @_LAYOUTS + def test_dispatch_command_emits_flags_after_model( + self, runs, skills_mode, dispatch_head + ): + result = self._copilot(skills_mode).dispatch_command( + "speckit.plan", + "body", + model="gpt-5", + stream=False, + reasoning_effort="xhigh", + context_size="long_context", + ) + assert result["exit_code"] == 0 + assert runs == [ + [ + *dispatch_head, + "--model", "gpt-5", + "--reasoning-effort", "xhigh", + "--context", "long_context", + "--output-format", "json", + ] + ] + + @_LAYOUTS + def test_unset_settings_leave_argv_unchanged( + self, runs, skills_mode, dispatch_head + ): + copilot = self._copilot(skills_mode) + baseline = [ + "copilot", "-p", "do it", "--model", "gpt-5", "--output-format", "json" + ] + assert copilot.build_exec_args("do it", model="gpt-5") == baseline + assert copilot.build_exec_args( + "do it", model="gpt-5", reasoning_effort=None, context_size=None + ) == baseline + + copilot.dispatch_command("speckit.plan", "body", model="gpt-5", stream=False) + copilot.dispatch_command( + "speckit.plan", + "body", + model="gpt-5", + stream=False, + reasoning_effort=None, + context_size=None, + ) + expected = [*dispatch_head, "--model", "gpt-5", "--output-format", "json"] + assert runs == [expected, expected] + + def test_accepted_values_match_copilot_cli(self): + assert REASONING_EFFORT_LEVELS == ( + "none", "minimal", "low", "medium", "high", "xhigh", "max" + ) + assert CONTEXT_TIERS == ("default", "long_context") + + @_ACCEPTED_VALUES + def test_every_accepted_value_reaches_the_cli(self, runs, setting, flag, value): + copilot = self._copilot() + assert copilot.build_exec_args("do it", **{setting: value}) == [ + "copilot", "-p", "do it", flag, value, "--output-format", "json" + ] + copilot.dispatch_command( + "speckit.plan", "body", stream=False, **{setting: value} + ) + assert runs == [ + ["copilot", "-p", "/speckit-plan body", flag, value, + "--output-format", "json"] + ] + + @pytest.mark.usefixtures("no_runs") + @pytest.mark.parametrize( + ("setting", "value", "accepted", "hatch"), + [ + ("reasoning_effort", "bogus", REASONING_EFFORT_LEVELS, "--reasoning-effort"), + ("reasoning_effort", "HIGH", REASONING_EFFORT_LEVELS, "--reasoning-effort"), + ("reasoning_effort", _SECRET_OUTPUT, REASONING_EFFORT_LEVELS, "--reasoning-effort"), + ("context_size", "huge", CONTEXT_TIERS, "--context"), + ("context_size", "200000", CONTEXT_TIERS, "--context"), + ("context_size", _SECRET_OUTPUT, CONTEXT_TIERS, "--context"), + ], + ) + def test_unknown_values_fail_before_the_cli_starts( + self, setting, value, accepted, hatch + ): + copilot = self._copilot() + with pytest.raises(DispatchSettingsError) as built: + copilot.build_exec_args("do it", **{setting: value}) + with pytest.raises(DispatchSettingsError) as dispatched: + copilot.dispatch_command("speckit.plan", stream=False, **{setting: value}) + for excinfo in (built, dispatched): + message = str(excinfo.value) + assert excinfo.value.setting == setting + assert value not in message + assert f"must be one of {', '.join(accepted)}." in message + assert ( + f'SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS="{hatch} "' in message + ) + + def test_extra_args_escape_hatch_passes_newer_values(self, monkeypatch): + monkeypatch.setenv( + "SPECKIT_INTEGRATION_COPILOT_EXTRA_ARGS", "--reasoning-effort ultra" + ) + assert self._copilot().build_exec_args("do it") == [ + "copilot", "-p", "do it", "--reasoning-effort", "ultra", + "--output-format", "json", + ] + + @pytest.mark.usefixtures("no_runs") + @pytest.mark.parametrize( + ("setting", "value"), + [ + ("reasoning_effort", ""), + ("context_size", ""), + ("reasoning_effort", "--yolo"), + ("context_size", "long context"), + ], + ) + def test_direct_calls_reject_malformed_values(self, setting, value): + copilot = self._copilot() + with pytest.raises(DispatchSettingsError) as built: + copilot.build_exec_args("do it", **{setting: value}) + with pytest.raises(DispatchSettingsError) as dispatched: + copilot.dispatch_command("speckit.plan", stream=False, **{setting: value}) + assert built.value.setting == setting + assert dispatched.value.setting == setting + + def test_legacy_builder_subclass_keeps_the_command_path(self, runs): + legacy = self._copilot(cls=_LegacyBuilderCopilot) + settings = {"reasoning_effort": "high", "context_size": "long_context"} + + assert effective_dispatch_settings( + legacy, via="dispatch_command" + ) == frozenset(settings) + check_dispatch_settings(legacy, settings, via="dispatch_command") + legacy.dispatch_command("speckit.plan", "body", stream=False, **settings) + assert runs == [ + [ + "copilot", "-p", "/speckit-plan body", + "--reasoning-effort", "high", + "--context", "long_context", + "--output-format", "json", + ] + ] + + def test_legacy_builder_subclass_rejects_the_prompt_path(self): + legacy = self._copilot(cls=_LegacyBuilderCopilot) + + assert effective_dispatch_settings(legacy, via="build_exec_args") == frozenset() + with pytest.raises( + DispatchSettingsError, + match="does not support per-step 'reasoning_effort'", + ) as excinfo: + check_dispatch_settings( + legacy, {"reasoning_effort": "high"}, via="build_exec_args" + ) + assert excinfo.value.setting == "reasoning_effort" + + @pytest.mark.parametrize( + ("step", "config", "prompt"), + [(*_COMMAND_STEP, "/speckit-plan body"), (*_PROMPT_STEP, "Summarise")], + ids=["command", "prompt"], + ) + def test_workflow_steps_dispatch_settings( + self, monkeypatch, tmp_path, runs, step, config, prompt + ): + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + result = step().execute( + { + "id": "ask", + **config, + "reasoning_effort": "high", + "context_size": "long_context", + }, + StepContext(default_integration="copilot", project_root=str(tmp_path)), + ) + assert result.status == StepStatus.COMPLETED, result.error + assert result.output["reasoning_effort"] == "high" + assert result.output["context_size"] == "long_context" + (args,) = runs + assert args[1:] == [ + "-p", prompt, "--reasoning-effort", "high", "--context", "long_context" + ] + + @pytest.mark.parametrize("setting", ["reasoning_effort", "context_size"]) + @pytest.mark.parametrize( + ("label", "step"), + [("Command", _COMMAND_STEP[1]), ("Prompt", _PROMPT_STEP[1])], + ids=["command", "prompt"], + ) + def test_captured_secret_stays_out_of_the_cli_error_and_log( + self, monkeypatch, tmp_path, label, step, setting + ): + """A streaming step that reads captured output into a setting fails + before Copilot CLI could repeat the value.""" + captured, streamed = [], [] + + def run_captured(args, **kwargs): + captured.append(list(args)) + return CapturedProcess(exit_code=0, stdout=_SECRET_OUTPUT, stderr="") + + def run(args, **kwargs): + streamed.append(list(args)) + return subprocess.CompletedProcess(args, 1, stdout="", stderr="") + + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: f"/bin/{name}") + monkeypatch.setattr( + "specify_cli.workflows.step.prompt.run_captured", run_captured + ) + monkeypatch.setattr(subprocess, "run", run) + (tmp_path / ".specify" / "workflows").mkdir(parents=True) + definition = WorkflowDefinition( + { + "schema_version": "1.0", + "workflow": { + "id": "leak", + "name": "leak", + "version": "1.0.0", + "integration": "copilot", + }, + "steps": [ + { + "id": "read", + "type": "prompt", + "prompt": "Print the token", + "capture": True, + }, + {"id": "ask", **step, setting: "{{ steps.read.output.stdout }}"}, + ], + } + ) + + state = WorkflowEngine(tmp_path).execute(definition) + + assert state.step_results["read"]["output"]["stdout"] == _SECRET_OUTPUT + assert len(captured) == 1 + assert streamed == [] + assert state.status is RunStatus.FAILED + error = state.step_results["ask"]["error"] + assert error.startswith( + f"{label} step 'ask': GitHub Copilot '{setting}' must be one of " + ) + assert error.endswith(" (set by step)") + assert _SECRET_OUTPUT not in error + log = ( + tmp_path / ".specify" / "workflows" / "runs" / state.run_id / "log.jsonl" + ).read_text(encoding="utf-8") + assert _SECRET_OUTPUT not in log