diff --git a/.changeset/patch-escape-mcp-env-secrets.md b/.changeset/patch-escape-mcp-env-secrets.md new file mode 100644 index 00000000000..5320cac9327 --- /dev/null +++ b/.changeset/patch-escape-mcp-env-secrets.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Escape MCP custom env secrets for non-Copilot engines so heredoc JSON stays valid during gateway startup. diff --git a/.github/workflows/daily-reliability-review.lock.yml b/.github/workflows/daily-reliability-review.lock.yml index 7182d2499c5..c362ec0d28e 100644 --- a/.github/workflows/daily-reliability-review.lock.yml +++ b/.github/workflows/daily-reliability-review.lock.yml @@ -733,7 +733,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -e SENTRY_ACCESS_TOKEN -e SENTRY_HOST -e SENTRY_OPENAI_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.7' GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_3af36bbf10eee298_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_05bb933389d186ee_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "safeoutputs": { @@ -794,9 +794,9 @@ jobs: "get_doc" ], "env": { - "OPENAI_API_KEY": "${SENTRY_OPENAI_API_KEY}", - "SENTRY_ACCESS_TOKEN": "${SENTRY_ACCESS_TOKEN}", - "SENTRY_HOST": "${{ env.SENTRY_HOST || 'sentry.io' }}" + "OPENAI_API_KEY": "\${SENTRY_OPENAI_API_KEY}", + "SENTRY_ACCESS_TOKEN": "\${SENTRY_ACCESS_TOKEN}", + "SENTRY_HOST": "\${SENTRY_HOST}" }, "guard-policies": { "write-sink": { @@ -821,7 +821,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_3af36bbf10eee298_EOF + GH_AW_MCP_CONFIG_05bb933389d186ee_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true diff --git a/.github/workflows/daily-token-consumption-report.lock.yml b/.github/workflows/daily-token-consumption-report.lock.yml index c5cc3f1ec65..eac83eee0cb 100644 --- a/.github/workflows/daily-token-consumption-report.lock.yml +++ b/.github/workflows/daily-token-consumption-report.lock.yml @@ -771,7 +771,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -e GRAFANA_SERVICE_ACCOUNT_TOKEN -e GRAFANA_URL -e SENTRY_ACCESS_TOKEN -e SENTRY_HOST -e SENTRY_OPENAI_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.7' GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_b99a9a3c8e270cf7_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_b6c1f98987f65dc9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -808,8 +808,8 @@ jobs: "tempo_docs-traceql" ], "env": { - "GRAFANA_SERVICE_ACCOUNT_TOKEN": "${GRAFANA_SERVICE_ACCOUNT_TOKEN}", - "GRAFANA_URL": "${GRAFANA_URL}" + "GRAFANA_SERVICE_ACCOUNT_TOKEN": "\${GRAFANA_SERVICE_ACCOUNT_TOKEN}", + "GRAFANA_URL": "\${GRAFANA_URL}" }, "guard-policies": { "write-sink": { @@ -878,9 +878,9 @@ jobs: "get_doc" ], "env": { - "OPENAI_API_KEY": "${SENTRY_OPENAI_API_KEY}", - "SENTRY_ACCESS_TOKEN": "${SENTRY_ACCESS_TOKEN}", - "SENTRY_HOST": "${{ env.SENTRY_HOST || 'sentry.io' }}" + "OPENAI_API_KEY": "\${SENTRY_OPENAI_API_KEY}", + "SENTRY_ACCESS_TOKEN": "\${SENTRY_ACCESS_TOKEN}", + "SENTRY_HOST": "\${SENTRY_HOST}" }, "guard-policies": { "write-sink": { @@ -905,7 +905,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_b99a9a3c8e270cf7_EOF + GH_AW_MCP_CONFIG_b6c1f98987f65dc9_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index e191bf67f5a..48c34b43947 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -1123,7 +1123,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -e GH_AW_WORKFLOW_ID_SANITIZED -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.7' GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_4efae7eb010ef26e_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_eb3b83d1e28e221d_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "agentdb": { @@ -1139,7 +1139,7 @@ jobs: "*" ], "env": { - "AGENTDB_PATH": "/tmp/gh-aw/cache-memory/agentdb-${{ env.GH_AW_WORKFLOW_ID_SANITIZED }}/discussions.rvf" + "AGENTDB_PATH": "/tmp/gh-aw/cache-memory/agentdb-\${GH_AW_WORKFLOW_ID_SANITIZED}/discussions.rvf" }, "guard-policies": { "write-sink": { @@ -1215,7 +1215,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_4efae7eb010ef26e_EOF + GH_AW_MCP_CONFIG_eb3b83d1e28e221d_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 395c8f1d9cf..a333400db3f 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -780,7 +780,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e GRAFANA_SERVICE_ACCOUNT_TOKEN -e GRAFANA_URL -e SENTRY_ACCESS_TOKEN -e SENTRY_HOST -e SENTRY_OPENAI_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.7' GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_fc7e9b442153deb2_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_7b19610baf738b58_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -817,8 +817,8 @@ jobs: "tempo_docs-traceql" ], "env": { - "GRAFANA_SERVICE_ACCOUNT_TOKEN": "${GRAFANA_SERVICE_ACCOUNT_TOKEN}", - "GRAFANA_URL": "${GRAFANA_URL}" + "GRAFANA_SERVICE_ACCOUNT_TOKEN": "\${GRAFANA_SERVICE_ACCOUNT_TOKEN}", + "GRAFANA_URL": "\${GRAFANA_URL}" }, "guard-policies": { "write-sink": { @@ -887,9 +887,9 @@ jobs: "get_doc" ], "env": { - "OPENAI_API_KEY": "${SENTRY_OPENAI_API_KEY}", - "SENTRY_ACCESS_TOKEN": "${SENTRY_ACCESS_TOKEN}", - "SENTRY_HOST": "${{ env.SENTRY_HOST || 'sentry.io' }}" + "OPENAI_API_KEY": "\${SENTRY_OPENAI_API_KEY}", + "SENTRY_ACCESS_TOKEN": "\${SENTRY_ACCESS_TOKEN}", + "SENTRY_HOST": "\${SENTRY_HOST}" }, "guard-policies": { "write-sink": { @@ -909,7 +909,7 @@ jobs: "startupTimeout": 120 } } - GH_AW_MCP_CONFIG_fc7e9b442153deb2_EOF + GH_AW_MCP_CONFIG_7b19610baf738b58_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true diff --git a/cmd/gh-aw/format_list_test.go b/cmd/gh-aw/format_list_test.go index 7551593489e..34a4be74c01 100644 --- a/cmd/gh-aw/format_list_test.go +++ b/cmd/gh-aw/format_list_test.go @@ -46,7 +46,6 @@ func TestFormatListWithOr(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() result := formatListWithOr(tt.items) diff --git a/pkg/workflow/README.md b/pkg/workflow/README.md index d57aba16f09..dd3161cac79 100644 --- a/pkg/workflow/README.md +++ b/pkg/workflow/README.md @@ -1540,7 +1540,6 @@ This appendix is generated from the current non-test Go source files in this pac | `secret_extraction.go` | `ExtractEnvExpressionsFromValue` | `func ExtractEnvExpressionsFromValue(value string) map[string]string` | ExtractEnvExpressionsFromValue extracts all GitHub Actions env expressions from a string value Returns a map of environment variable names to their full env expressions Examples: - "${{ env. | | `secret_extraction.go` | `ExtractSecretsFromMap` | `func ExtractSecretsFromMap(values map[string]string) map[string]string` | ExtractSecretsFromMap extracts all secrets from a map of string values Returns a map of environment variable names to their full secret expressions Example: Input: {"DD_API_KEY": "${{ secrets. | | `secret_extraction.go` | `ExtractWorkflowInputExpressionsFromValue` | `func ExtractWorkflowInputExpressionsFromValue(value string) map[string]string` | ExtractWorkflowInputExpressionsFromValue extracts simple workflow input expressions from a string value and maps them to deterministic environment variable names. | -| `secret_extraction.go` | `ReplaceSecretsWithBashVars` | `func ReplaceSecretsWithBashVars(value string) string` | ReplaceSecretsWithBashVars replaces secret expressions in a value with bash env var references. | | `secret_extraction.go` | `ReplaceTemplateExpressionsWithEnvVars` | `func ReplaceTemplateExpressionsWithEnvVars(value string) string` | ReplaceTemplateExpressionsWithEnvVars replaces all template expressions with environment variable references Handles: secrets. | | `secret_masking.go` | `(*Compiler).MergeSecretMasking` | `func (*Compiler).MergeSecretMasking(topConfig *SecretMaskingConfig, importedSecretMaskingJSON string) (*SecretMaskingConfig, error)` | MergeSecretMasking merges secret-masking configurations from imports with top-level config | | `service_ports.go` | `ExtractServicePortExpressions` | `func ExtractServicePortExpressions(servicesYAML string) (string, []string)` | ExtractServicePortExpressions parses the services: YAML string from WorkflowData. | diff --git a/pkg/workflow/mcp_config_copilot_test.go b/pkg/workflow/mcp_config_copilot_test.go index 5d953df9320..36c1b0a8e5f 100644 --- a/pkg/workflow/mcp_config_copilot_test.go +++ b/pkg/workflow/mcp_config_copilot_test.go @@ -3,6 +3,8 @@ package workflow import ( + "encoding/json" + "regexp" "strings" "testing" ) @@ -203,6 +205,164 @@ func TestRenderSharedMCPConfig_ToolsFieldGeneration(t *testing.T) { } } +// TestRenderCustomMCPEnvVars_NonCopilotSecretsEscaped verifies that for non-Copilot +// JSON engines, secrets in custom MCP server env blocks are rendered as \${VAR} +// (backslash-escaped) rather than ${VAR} (unescaped). Unescaped references would +// be expanded by bash inside the unquoted heredoc that carries the MCP gateway +// JSON config -- a secret containing '"' or '\' would corrupt the JSON and cause +// the gateway to fail before the agent runs. Backslash-escaping keeps the JSON +// valid regardless of the secret's runtime value. +func TestRenderCustomMCPEnvVars_NonCopilotSecretsEscaped(t *testing.T) { + tests := []struct { + name string + toolConfig map[string]any + renderer MCPConfigRenderer + expectedContent []string + unexpectedContent []string + // validateJSON, when true, replaces \${VAR} placeholders with a benign + // string (simulating bash heredoc stripping the leading backslash) and + // then verifies the result is parseable as a JSON object. This catches + // regressions where the rendered fragment would produce invalid JSON once + // the gateway resolves environment variables at runtime. + validateJSON bool + }{ + { + name: "Non-Copilot stdio container - secret in env uses backslash-escaped var", + toolConfig: map[string]any{ + "type": "stdio", + "container": "some/image:latest", + "env": map[string]any{ + "MY_TOKEN": "${{ secrets.MY_TOKEN }}", + }, + }, + renderer: MCPConfigRenderer{ + IndentLevel: " ", + Format: "json", + RequiresCopilotFields: false, + }, + // Secret must be rendered as \${MY_TOKEN} so the unquoted heredoc + // leaves a literal ${MY_TOKEN} string in the JSON (valid JSON). + expectedContent: []string{ + `"MY_TOKEN": "\${MY_TOKEN}"`, + }, + // Must NOT appear as an unescaped bash variable reference -- that + // would let bash splice the raw secret value into the JSON. + unexpectedContent: []string{ + `"MY_TOKEN": "${MY_TOKEN}"`, + }, + }, + { + name: "Copilot stdio - secret in env also uses backslash-escaped var", + toolConfig: map[string]any{ + "type": "stdio", + "container": "some/image:latest", + "env": map[string]any{ + "MY_TOKEN": "${{ secrets.MY_TOKEN }}", + }, + "allowed": []string{"*"}, + }, + renderer: MCPConfigRenderer{ + IndentLevel: " ", + Format: "json", + RequiresCopilotFields: true, + }, + expectedContent: []string{ + `"MY_TOKEN": "\${MY_TOKEN}"`, + }, + unexpectedContent: []string{ + `"MY_TOKEN": "${MY_TOKEN}"`, + }, + }, + { + name: "Non-Copilot stdio container - secret with fallback in env uses backslash-escaped var", + toolConfig: map[string]any{ + "type": "stdio", + "container": "some/image:latest", + "env": map[string]any{ + "DD_SITE": "${{ secrets.DD_SITE || 'datadoghq.com' }}", + }, + }, + renderer: MCPConfigRenderer{ + IndentLevel: " ", + Format: "json", + RequiresCopilotFields: false, + }, + expectedContent: []string{ + `"DD_SITE": "\${DD_SITE}"`, + }, + unexpectedContent: []string{ + `"DD_SITE": "${DD_SITE}"`, + }, + }, + { + // The var *name* is always a safe identifier, but this test verifies + // that the \${VAR} placeholder produces structurally valid JSON once + // the gateway resolves the env var at runtime. Before the fix, a bare + // ${VAR} would have been spliced directly by bash into the heredoc; + // any secret containing '"' or '\' would have corrupted the JSON. + name: "Non-Copilot stdio - env placeholder produces valid JSON after gateway resolution", + toolConfig: map[string]any{ + "type": "stdio", + "container": "some/image:latest", + "env": map[string]any{ + "SPECIAL_KEY": `${{ secrets.SPECIAL_KEY }}`, + }, + }, + renderer: MCPConfigRenderer{ + IndentLevel: " ", + Format: "json", + RequiresCopilotFields: false, + }, + expectedContent: []string{ + `"SPECIAL_KEY": "\${SPECIAL_KEY}"`, + }, + unexpectedContent: []string{ + `"SPECIAL_KEY": "${SPECIAL_KEY}"`, + }, + validateJSON: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var output strings.Builder + + err := renderSharedMCPConfig(&output, "test-tool", tt.toolConfig, tt.renderer) + if err != nil { + t.Fatalf("renderSharedMCPConfig failed: %v", err) + } + + result := output.String() + + for _, expected := range tt.expectedContent { + if !strings.Contains(result, expected) { + t.Errorf("Expected content not found: %q\nActual output:\n%s", expected, result) + } + } + + for _, unexpected := range tt.unexpectedContent { + if strings.Contains(result, unexpected) { + t.Errorf("Unexpected content found: %q\nActual output:\n%s", unexpected, result) + } + } + + if tt.validateJSON { + // Simulate bash heredoc processing (\${VAR} → ${VAR}) and then + // gateway env-var substitution (${VAR} → a benign placeholder). + // The resulting fragment must be parseable as a JSON object, + // proving that no secret value -- regardless of its content -- + // can corrupt the JSON structure. + escapedVarRe := regexp.MustCompile(`\\\$\{[A-Z0-9_]+\}`) + resolved := escapedVarRe.ReplaceAllString(result, "placeholder-value") + var obj map[string]any + if err := json.Unmarshal([]byte("{"+resolved+"}"), &obj); err != nil { + t.Errorf("Rendered output is not valid JSON after placeholder substitution: %v\nResolved fragment:\n%s", err, resolved) + } + } + }) + } +} + func TestRenderSharedMCPConfig_TypeConversion(t *testing.T) { tests := []struct { name string diff --git a/pkg/workflow/mcp_config_custom.go b/pkg/workflow/mcp_config_custom.go index b4f2d05c575..ccbe1b92230 100644 --- a/pkg/workflow/mcp_config_custom.go +++ b/pkg/workflow/mcp_config_custom.go @@ -54,10 +54,16 @@ func renderCustomMCPConfigWrapperWithContext(yaml *strings.Builder, toolName str // // For TOML output, GitHub Actions template expressions are rewritten to direct // ${VAR} references because Codex config expects shell-style environment -// expansion. For JSON output, Copilot uses escaped \${VAR} passthrough syntax, -// while non-Copilot engines use bash variable substitution to avoid embedding -// secret expressions directly in the generated run block. -func renderCustomMCPEnvVars(env map[string]string, tomlFormat, requiresCopilotFields bool) map[string]string { +// expansion. For JSON output, both Copilot and non-Copilot engines use the +// escaped \${VAR} passthrough syntax. The MCP gateway JSON config is written +// inside an unquoted heredoc, so any unescaped ${VAR} reference would be +// expanded by bash before the gateway sees it — splicing the raw secret bytes +// into JSON text. A secret value containing a '"' or '\' character would +// corrupt the JSON. Backslash-escaping (\${VAR}) prevents the heredoc from +// expanding the variable; bash only strips the leading backslash, leaving the +// literal ${VAR} string in the JSON, which the gateway then resolves safely +// from its own environment (RGS-008 compliance). +func renderCustomMCPEnvVars(env map[string]string, tomlFormat bool) map[string]string { renderedEnv := make(map[string]string, len(env)) for envKey, envValue := range env { if tomlFormat { @@ -67,14 +73,12 @@ func renderCustomMCPEnvVars(env map[string]string, tomlFormat, requiresCopilotFi envValue = strings.ReplaceAll(envValue, "${{ env.", "${") envValue = strings.ReplaceAll(envValue, "${{ github.workspace }}", "${GITHUB_WORKSPACE}") envValue = strings.ReplaceAll(envValue, " }}", "}") - } else if requiresCopilotFields { - // For Copilot, replace all template expressions with \${VAR} syntax. - envValue = ReplaceTemplateExpressionsWithEnvVars(envValue) } else { - // For non-Copilot engines, replace secrets with ${VAR} bash expansion so - // they are never directly interpolated in the run block (RGS-008). The - // env vars are injected into the step env block by collectMCPEnvironmentVariables. - envValue = ReplaceSecretsWithBashVars(envValue) + // For both Copilot and non-Copilot JSON engines, replace all template + // expressions with \${VAR} passthrough syntax. This keeps raw secret values + // out of the heredoc (RGS-008) and produces valid JSON regardless of the + // characters contained in the secret at runtime. + envValue = ReplaceTemplateExpressionsWithEnvVars(envValue) } renderedEnv[envKey] = envValue } @@ -338,7 +342,7 @@ func renderMCPMapProperty(yaml *strings.Builder, property string, isLast bool, m } func renderMCPEnvMap(yaml *strings.Builder, isLast bool, mcpConfig *parser.RegistryMCPServerConfig, renderer MCPConfigRenderer, headerSecrets map[string]string) { - renderedEnv := renderCustomMCPEnvVars(mcpConfig.Env, renderer.Format == "toml", renderer.RequiresCopilotFields) + renderedEnv := renderCustomMCPEnvVars(mcpConfig.Env, renderer.Format == "toml") if renderer.Format == "toml" { writeTOMLInlineStringMapSection(yaml, renderer.IndentLevel, "env", renderedEnv) return diff --git a/pkg/workflow/mcp_http_headers_test.go b/pkg/workflow/mcp_http_headers_test.go index 2bbb39b6edd..10c9598f349 100644 --- a/pkg/workflow/mcp_http_headers_test.go +++ b/pkg/workflow/mcp_http_headers_test.go @@ -144,44 +144,6 @@ func TestReplaceSecretsWithEnvVars(t *testing.T) { } } -func TestReplaceSecretsWithBashVars(t *testing.T) { - tests := []struct { - name string - value string - expected string - }{ - { - name: "Replace single secret", - value: "${{ secrets.SENTRY_ACCESS_TOKEN }}", - expected: "${SENTRY_ACCESS_TOKEN}", - }, - { - name: "Replace secret with default", - value: "${{ secrets.DD_SITE || 'datadoghq.com' }}", - expected: "${DD_SITE}", - }, - { - name: "Replace in Bearer token", - value: "Bearer ${{ secrets.TOKEN }}", - expected: "Bearer ${TOKEN}", - }, - { - name: "No replacement needed", - value: "static-value", - expected: "static-value", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := ReplaceSecretsWithBashVars(tt.value) - if result != tt.expected { - t.Errorf("Expected %q, got %q", tt.expected, result) - } - }) - } -} - func TestRenderSharedMCPConfig_HTTPWithHeaderSecrets(t *testing.T) { toolConfig := map[string]any{ "type": "http", diff --git a/pkg/workflow/secret_extraction.go b/pkg/workflow/secret_extraction.go index e5215ac0e04..c71b7f1a46b 100644 --- a/pkg/workflow/secret_extraction.go +++ b/pkg/workflow/secret_extraction.go @@ -160,23 +160,6 @@ func ReplaceSecretsWithEnvVars(value string, secrets map[string]string) string { return result } -// ReplaceSecretsWithBashVars replaces secret expressions in a value with bash env var references. -// Example: "${{ secrets.DD_API_KEY }}" -> "${DD_API_KEY}" -// Unlike ReplaceSecretsWithEnvVars, this does NOT add a backslash prefix, so bash expands -// the variable at runtime. Used for non-Copilot MCP server env blocks where the step env block -// already holds the corresponding env vars (injected by collectMCPEnvironmentVariables), -// preventing direct secret interpolation in run blocks (RGS-008 compliance). -func ReplaceSecretsWithBashVars(value string) string { - result := value - secrets := ExtractSecretsFromValue(value) - // Sort keys for deterministic output; see ReplaceSecretsWithEnvVars for rationale. - for _, varName := range sliceutil.SortedKeys(secrets) { - secretExpr := secrets[varName] - result = strings.ReplaceAll(result, secretExpr, "${"+varName+"}") - } - return result -} - // ExtractEnvExpressionsFromValue extracts all GitHub Actions env expressions from a string value // Returns a map of environment variable names to their full env expressions // Examples: diff --git a/pkg/workflow/secret_extraction_test.go b/pkg/workflow/secret_extraction_test.go index 708d85ea5f3..c03f980941d 100644 --- a/pkg/workflow/secret_extraction_test.go +++ b/pkg/workflow/secret_extraction_test.go @@ -372,33 +372,6 @@ func TestReplaceSecretsWithEnvVars_FallbackExpressionDeterminism(t *testing.T) { } } -// TestReplaceSecretsWithBashVars_FallbackExpressionDeterminism verifies that -// ReplaceSecretsWithBashVars produces deterministic output for fallback expressions -// with two secret references. Mirrors TestReplaceSecretsWithEnvVars_FallbackExpressionDeterminism. -func TestReplaceSecretsWithBashVars_FallbackExpressionDeterminism(t *testing.T) { - value := "${{ secrets.DD_APPLICATION_KEY || secrets.DD_APP_KEY }}" - - const runs = 50 - var first string - for i := range runs { - got := ReplaceSecretsWithBashVars(value) - if i == 0 { - first = got - continue - } - if got != first { - t.Errorf("non-deterministic output: run 0 produced %q, run %d produced %q", first, i, got) - } - } - - // "DD_APPLICATION_KEY" sorts before "DD_APP_KEY" ('L' < '_'), so it wins. - want := "${DD_APPLICATION_KEY}" - if first != want { - t.Errorf("ReplaceSecretsWithBashVars(%q) = %q, want %q", value, first, want) - } -} - -// TestSharedExtractSecretsFromValueEdgeCases tests edge cases for the shared ExtractSecretsFromValue utility function func TestSharedExtractSecretsFromValueEdgeCases(t *testing.T) { tests := []struct { name string