From acdaa2bfc30fadb9c4be7364373614f557528010 Mon Sep 17 00:00:00 2001 From: vitek-karas <10670590+vitek-karas@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:30:23 +0200 Subject: [PATCH 1/3] Replace CI scan curl access Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-evidence-reader | 652 ++++++++++++++++++ .github/workflows/ci-failure-scan.lock.yml | 11 +- .github/workflows/ci-failure-scan.md | 45 +- .../tests/test_ci_evidence_reader.py | 263 +++++++ 4 files changed, 946 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/ci-evidence-reader create mode 100644 .github/workflows/tests/test_ci_evidence_reader.py diff --git a/.github/workflows/ci-evidence-reader b/.github/workflows/ci-evidence-reader new file mode 100644 index 00000000000000..e112197bdbb7c7 --- /dev/null +++ b/.github/workflows/ci-evidence-reader @@ -0,0 +1,652 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import re +import secrets +import stat +import sys +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + + +OUTPUT_ROOT = Path("/tmp/gh-aw/agent") +TIMEOUT_SECONDS = 30 +JSON_LIMIT = 16 * 1024 * 1024 +LOG_LIMIT = 64 * 1024 * 1024 +CHUNK_SIZE = 64 * 1024 +DEFINITION_IDS = ( + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 123, + 124, + 134, + 136, + 137, + 138, + 140, + 141, + 144, + 145, + 146, + 150, + 153, + 154, + 155, + 159, + 160, + 230, + 235, + 265, + 309, + 316, + 330, +) +ALLOWED_SKIPS = (0, 10, 20, 30, 40) +AZDO_HOST = "dev.azure.com" +AZDO_PATH_PREFIX = "/dnceng-public/public/_apis/build/" +HELIX_HOST = "helix.dot.net" +USER_AGENT = "ci-evidence-reader/1.0" + +_AZDO_BUILD_PATH = re.compile(r"^/dnceng-public/public/_apis/build/builds$") +_AZDO_TIMELINE_PATH = re.compile( + r"^/dnceng-public/public/_apis/build/builds/[1-9][0-9]*/timeline$" +) +_AZDO_LOG_PATH = re.compile( + r"^/dnceng-public/public/_apis/build/builds/[1-9][0-9]*/logs/[1-9][0-9]*$" +) +_HELIX_WORK_ITEMS_PATH = re.compile( + r"^/api/jobs/[0-9a-f-]{36}/workitems$", re.IGNORECASE +) +_HELIX_CONSOLE_PATH = re.compile( + r"^/api/(?:2019-06-17/)?jobs/[0-9a-f-]{36}/workitems/" + r"[^/]+/files/console(?:\.[A-Za-z0-9_-]+)?\.log$", + re.IGNORECASE, +) +_BLOB_CONSOLE_PATH = re.compile( + r"^/(?:[^/]+/)+console(?:\.[A-Za-z0-9_-]+)?\.log$", + re.IGNORECASE, +) +_WORK_ITEM_NAME = re.compile(r"^[^\x00-\x1f\x7f]{1,512}$") +_BLOB_QUERY_KEYS = { + "helixlogtype", + "rscc", + "rscd", + "rsce", + "rscl", + "rsct", + "se", + "si", + "sig", + "sip", + "ske", + "skoid", + "sks", + "skt", + "sktid", + "skv", + "sp", + "spr", + "sr", + "st", + "sv", +} + + +class TransportError(RuntimeError): + pass + + +def _single_query_values(url: str) -> tuple[urllib.parse.SplitResult, dict[str, str]]: + parts = urllib.parse.urlsplit(url) + if parts.scheme != "https": + raise TransportError("only HTTPS URLs are allowed") + if parts.username is not None or parts.password is not None: + raise TransportError("URL credentials are not allowed") + if parts.fragment: + raise TransportError("URL fragments are not allowed") + try: + if parts.port not in (None, 443): + raise TransportError("only the default HTTPS port is allowed") + except ValueError as error: + raise TransportError("invalid URL port") from error + + values: dict[str, str] = {} + try: + query_values = ( + urllib.parse.parse_qsl( + parts.query, keep_blank_values=True, strict_parsing=True + ) + if parts.query + else () + ) + except ValueError as error: + raise TransportError("invalid URL query") from error + for key, value in query_values: + if key in values: + raise TransportError(f"duplicate query parameter: {key}") + values[key] = value + return parts, values + + +def _validate_azdo_url(parts: urllib.parse.SplitResult, query: dict[str, str]) -> None: + if parts.hostname != AZDO_HOST or not parts.path.startswith(AZDO_PATH_PREFIX): + raise TransportError("URL is not a permitted dnceng-public build API endpoint") + + if _AZDO_BUILD_PATH.fullmatch(parts.path): + expected_keys = { + "api-version", + "branchName", + "definitions", + "resultFilter", + "statusFilter", + "$top", + "$skip", + } + if set(query) != expected_keys: + raise TransportError("unexpected Azure DevOps build-list query parameters") + try: + definition = int(query["definitions"]) + top = int(query["$top"]) + except ValueError as error: + raise TransportError("invalid Azure DevOps build-list query") from error + try: + skip = int(query["$skip"]) + except ValueError as error: + raise TransportError("invalid Azure DevOps build-list query") from error + if ( + definition not in DEFINITION_IDS + or top != 25 + or skip not in ALLOWED_SKIPS + ): + raise TransportError("Azure DevOps build-list query is outside permitted bounds") + if ( + query["api-version"] != "7.1" + or query["branchName"] != "refs/heads/main" + or query["statusFilter"] != "completed" + or query["resultFilter"] != "succeeded,failed,partiallySucceeded" + ): + raise TransportError("Azure DevOps build-list filters are not permitted") + return + + if ( + _AZDO_TIMELINE_PATH.fullmatch(parts.path) + or _AZDO_LOG_PATH.fullmatch(parts.path) + ) and query == {"api-version": "7.1"}: + return + + raise TransportError("URL is not a permitted Azure DevOps build endpoint") + + +def _validate_helix_api_url( + parts: urllib.parse.SplitResult, query: dict[str, str] +) -> str: + if parts.hostname != HELIX_HOST: + raise TransportError("URL is not a permitted Helix endpoint") + if _HELIX_WORK_ITEMS_PATH.fullmatch(parts.path) and query == { + "api-version": "2019-06-17" + }: + return "helix-work-items" + if _HELIX_CONSOLE_PATH.fullmatch(parts.path) and not query: + return "helix-console" + raise TransportError("URL is not a permitted Helix endpoint") + + +def _validate_blob_console_url( + parts: urllib.parse.SplitResult, query: dict[str, str] +) -> None: + hostname = parts.hostname or "" + if ( + not hostname.startswith("helix") + or not hostname.endswith(".blob.core.windows.net") + or not _BLOB_CONSOLE_PATH.fullmatch(parts.path) + ): + raise TransportError("URL is not a permitted Helix console blob") + if not set(query).issubset(_BLOB_QUERY_KEYS): + raise TransportError("unexpected Helix console blob query parameters") + + +def _validate_url(url: str, allowed_families: set[str]) -> None: + parts, query = _single_query_values(url) + if parts.hostname == AZDO_HOST: + _validate_azdo_url(parts, query) + family = "azdo" + elif parts.hostname == HELIX_HOST: + family = _validate_helix_api_url(parts, query) + elif (parts.hostname or "").endswith(".blob.core.windows.net"): + _validate_blob_console_url(parts, query) + family = "helix-console" + else: + raise TransportError("URL host is not permitted") + + if family not in allowed_families: + raise TransportError(f"redirect escaped the permitted {sorted(allowed_families)} endpoints") + + +class _ValidatingRedirectHandler(urllib.request.HTTPRedirectHandler): + def __init__(self, allowed_families: set[str]) -> None: + self._allowed_families = allowed_families + + def redirect_request(self, req, fp, code, msg, headers, newurl): + _validate_url(newurl, self._allowed_families) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _read_response(response, limit: int) -> bytes: + content_length = response.headers.get("Content-Length") + if content_length is not None: + try: + if int(content_length) > limit: + raise TransportError(f"response exceeds the {limit}-byte limit") + except ValueError as error: + raise TransportError("invalid Content-Length header") from error + + chunks: list[bytes] = [] + total = 0 + while True: + chunk = response.read(min(CHUNK_SIZE, limit - total + 1)) + if not chunk: + return b"".join(chunks) + total += len(chunk) + if total > limit: + raise TransportError(f"response exceeds the {limit}-byte limit") + chunks.append(chunk) + + +def _request_bytes( + url: str, + allowed_families: set[str], + limit: int, + opener=None, +) -> bytes: + _validate_url(url, allowed_families) + if opener is None: + opener = urllib.request.build_opener( + _ValidatingRedirectHandler(allowed_families) + ) + request = urllib.request.Request(url, method="GET", headers={"User-Agent": USER_AGENT}) + try: + with opener.open(request, timeout=TIMEOUT_SECONDS) as response: + status = getattr(response, "status", 200) + if not 200 <= status < 300: + raise TransportError(f"HTTP request failed with status {status}") + return _read_response(response, limit) + except urllib.error.HTTPError as error: + raise TransportError(f"HTTP request failed with status {error.code}") from error + except urllib.error.URLError as error: + raise TransportError(f"HTTP request failed: {error.reason}") from error + except TimeoutError as error: + raise TransportError("HTTP request timed out") from error + + +def _validate_output_path(value: str, suffixes: tuple[str, ...]) -> Path: + candidate = Path(value) + if not candidate.is_absolute(): + raise TransportError("output path must be absolute") + + root = Path(os.path.abspath(OUTPUT_ROOT)) + candidate = Path(os.path.abspath(candidate)) + try: + candidate.relative_to(root) + except ValueError as error: + raise TransportError(f"output path must be under {OUTPUT_ROOT}/") from error + + resolved_root = root.resolve() + resolved = candidate.resolve(strict=False) + try: + resolved.relative_to(resolved_root) + except ValueError as error: + raise TransportError(f"output path must be under {OUTPUT_ROOT}/") from error + if candidate == root or candidate.suffix.lower() not in suffixes: + raise TransportError(f"output path must end in one of: {', '.join(suffixes)}") + if candidate.is_symlink(): + raise TransportError("output path must not be a symlink") + if candidate.exists() and not candidate.is_file(): + raise TransportError("output path must be a regular file") + return candidate + + +def _open_output_parent(output: Path) -> tuple[int, str]: + root = Path(os.path.abspath(OUTPUT_ROOT)) + relative = output.relative_to(root) + root.mkdir(parents=True, exist_ok=True) + + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + current_fd = -1 + try: + current_fd = os.open(root, flags) + for component in relative.parts[:-1]: + try: + os.mkdir(component, mode=0o700, dir_fd=current_fd) + except FileExistsError: + pass + next_fd = os.open(component, flags, dir_fd=current_fd) + os.close(current_fd) + current_fd = next_fd + return current_fd, relative.name + except OSError as error: + if current_fd >= 0: + os.close(current_fd) + raise TransportError("output parent must be a real directory") from error + + +def _open_portable_output_parent(output: Path) -> Path: + root = Path(os.path.abspath(OUTPUT_ROOT)) + relative = output.relative_to(root) + try: + root.mkdir(parents=True, exist_ok=True) + except OSError as error: + raise TransportError("output parent must be a real directory") from error + if root.is_symlink() or not root.is_dir(): + raise TransportError("output parent must be a real directory") + + parent = root + for component in relative.parts[:-1]: + parent /= component + try: + parent.mkdir(mode=0o700, exist_ok=True) + except OSError as error: + raise TransportError("output parent must be a real directory") from error + if parent.is_symlink() or not parent.is_dir(): + raise TransportError("output parent must be a real directory") + return parent + + +def _reject_invalid_output(parent_fd: int, output_name: str) -> None: + try: + output_stat = os.stat(output_name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError: + return + if not stat.S_ISREG(output_stat.st_mode): + raise TransportError("output path must be a regular file") + + +def _write_output_portably(data: bytes, output: Path) -> None: + parent = _open_portable_output_parent(output) + temporary = parent / f".{output.name}.{secrets.token_hex(8)}.tmp" + try: + if output.is_symlink() or (output.exists() and not output.is_file()): + raise TransportError("output path must be a regular file") + descriptor = os.open( + temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600 + ) + with os.fdopen(descriptor, "wb") as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + if output.is_symlink() or (output.exists() and not output.is_file()): + raise TransportError("output path must be a regular file") + os.replace(temporary, output) + except OSError as error: + raise TransportError("could not write output") from error + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def _write_output(data: bytes, value: str, suffixes: tuple[str, ...]) -> None: + output = _validate_output_path(value, suffixes) + if not (hasattr(os, "O_DIRECTORY") and hasattr(os, "O_NOFOLLOW")): + _write_output_portably(data, output) + print(f"wrote {len(data)} bytes to {output}") + return + + parent_fd, output_name = _open_output_parent(output) + temporary_name = f".{output_name}.{secrets.token_hex(8)}.tmp" + try: + _reject_invalid_output(parent_fd, output_name) + descriptor = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=parent_fd, + ) + with os.fdopen(descriptor, "wb") as temporary: + temporary.write(data) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace( + temporary_name, + output_name, + src_dir_fd=parent_fd, + dst_dir_fd=parent_fd, + ) + except OSError as error: + try: + os.unlink(temporary_name, dir_fd=parent_fd) + except FileNotFoundError: + pass + raise TransportError("could not write output") from error + finally: + os.close(parent_fd) + print(f"wrote {len(data)} bytes to {output}") + + +def _positive_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("must be an integer") from error + if parsed <= 0: + raise argparse.ArgumentTypeError("must be positive") + return parsed + + +def _definition_id(value: str) -> int: + parsed = _positive_int(value) + if parsed not in DEFINITION_IDS: + raise argparse.ArgumentTypeError( + f"must be one of: {', '.join(str(item) for item in DEFINITION_IDS)}" + ) + return parsed + + +def _top(value: str) -> int: + try: + parsed = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("must be an integer") from error + if parsed != 25: + raise argparse.ArgumentTypeError("must be 25") + return parsed + + +def _skip(value: str) -> int: + try: + parsed = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("must be an integer") from error + if parsed not in ALLOWED_SKIPS: + raise argparse.ArgumentTypeError("must be one of: 0, 10, 20, 30, 40") + return parsed + + +def _job_id(value: str) -> str: + try: + return str(uuid.UUID(value)) + except ValueError as error: + raise argparse.ArgumentTypeError("must be a UUID") from error + + +def _work_item(value: str) -> str: + if not _WORK_ITEM_NAME.fullmatch(value) or value in {".", ".."}: + raise argparse.ArgumentTypeError("invalid work-item name") + return value + + +def _azdo_builds_url(definition: int, top: int = 25, skip: int = 0) -> str: + query = urllib.parse.urlencode( + { + "definitions": definition, + "branchName": "refs/heads/main", + "statusFilter": "completed", + "resultFilter": "succeeded,failed,partiallySucceeded", + "$top": top, + "$skip": skip, + "api-version": "7.1", + }, + safe=",/", + ) + return f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds?{query}" + + +def _helix_work_items_url(job_id: str) -> str: + return ( + f"https://{HELIX_HOST}/api/jobs/{job_id}/workitems" + "?api-version=2019-06-17" + ) + + +def _json_items(payload: bytes) -> list[dict]: + try: + document = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise TransportError("Helix work-item response was not valid JSON") from error + + if isinstance(document, list): + items = document + elif isinstance(document, dict): + items = next( + ( + document[key] + for key in ("value", "WorkItems", "workItems") + if isinstance(document.get(key), list) + ), + None, + ) + else: + items = None + if items is None or not all(isinstance(item, dict) for item in items): + raise TransportError("Helix work-item response had an unexpected shape") + return items + + +def _case_insensitive_field(item: dict, field_name: str): + for key, value in item.items(): + if key.casefold() == field_name.casefold(): + return value + return None + + +def _console_url(payload: bytes, work_item: str) -> str: + matches = [ + item + for item in _json_items(payload) + if ( + _case_insensitive_field(item, "Name") + or _case_insensitive_field(item, "WorkItemName") + ) + == work_item + ] + if len(matches) != 1: + raise TransportError( + f"expected exactly one Helix work item named {work_item!r}, found {len(matches)}" + ) + console_url = _case_insensitive_field(matches[0], "ConsoleOutputUri") + if not isinstance(console_url, str) or not console_url: + raise TransportError("Helix work item did not contain a console output URL") + _validate_url(console_url, {"helix-console"}) + return console_url + + +def _run(args: argparse.Namespace) -> None: + if args.command == "azdo-builds": + data = _request_bytes( + _azdo_builds_url(args.definition, args.top, args.skip), {"azdo"}, JSON_LIMIT + ) + _write_output(data, args.output, (".json",)) + elif args.command == "azdo-timeline": + url = ( + f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds/{args.build_id}/timeline" + "?api-version=7.1" + ) + _write_output( + _request_bytes(url, {"azdo"}, JSON_LIMIT), args.output, (".json",) + ) + elif args.command == "azdo-log": + url = ( + f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds/{args.build_id}/logs/" + f"{args.log_id}?api-version=7.1" + ) + _write_output( + _request_bytes(url, {"azdo"}, LOG_LIMIT), args.output, (".log", ".txt") + ) + elif args.command == "helix-work-items": + data = _request_bytes( + _helix_work_items_url(args.job_id), {"helix-work-items"}, JSON_LIMIT + ) + _write_output(data, args.output, (".json",)) + elif args.command == "helix-console": + work_items = _request_bytes( + _helix_work_items_url(args.job_id), {"helix-work-items"}, JSON_LIMIT + ) + console_url = _console_url(work_items, args.work_item) + _write_output( + _request_bytes(console_url, {"helix-console"}, LOG_LIMIT), + args.output, + (".log", ".txt"), + ) + else: + raise AssertionError(f"unhandled command: {args.command}") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Read constrained CI evidence from approved endpoints" + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + builds = subparsers.add_parser("azdo-builds") + builds.add_argument("--definition", required=True, type=_definition_id) + builds.add_argument("--top", type=_top, default=25) + builds.add_argument("--skip", type=_skip, default=0) + builds.add_argument("--output", required=True) + + timeline = subparsers.add_parser("azdo-timeline") + timeline.add_argument("--build-id", required=True, type=_positive_int) + timeline.add_argument("--output", required=True) + + log = subparsers.add_parser("azdo-log") + log.add_argument("--build-id", required=True, type=_positive_int) + log.add_argument("--log-id", required=True, type=_positive_int) + log.add_argument("--output", required=True) + + work_items = subparsers.add_parser("helix-work-items") + work_items.add_argument("--job-id", required=True, type=_job_id) + work_items.add_argument("--output", required=True) + + console = subparsers.add_parser("helix-console") + console.add_argument("--job-id", required=True, type=_job_id) + console.add_argument("--work-item", required=True, type=_work_item) + console.add_argument("--output", required=True) + return parser + + +def main() -> int: + try: + _run(_parser().parse_args()) + return 0 + except TransportError as error: + print(f"ci-evidence-reader: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index d9e5e4c61032f5..af2fba7f82f353 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"60fadff2edc8a0b8c29643c87487baa414197ae8232ea8ef018969e42bfc2c05","body_hash":"cb0e6f40f206b18d3fce5fb8da89c9f3be5eeb5bc43aabce6c850a5175f9f1cd","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"785d5920cc294c31c191bc526007c8325ee1d42a3f5827b1cff590032e5ad901","body_hash":"ec49d49010194546a715b5209474b666cc6d93ce6d18186324b3312735d5fb8b","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -508,6 +508,9 @@ jobs: env: GH_AW_SKILL_DIR: ".github/skills" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Install CI evidence reader + run: "mkdir -p \"${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin\"\ncp .github/workflows/ci-evidence-reader \"${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin/ci-evidence-reader\"\nchmod 0555 \"${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin/ci-evidence-reader\"\nprintf '%s\\n' \"${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin\" >> \"$GITHUB_PATH\"" + - name: Download container images run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7 ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627 ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196 ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e - name: Generate Safe Outputs Config @@ -780,7 +783,7 @@ jobs: # --allow-tool shell(bash) # --allow-tool shell(cat) # --allow-tool shell(chmod) - # --allow-tool shell(curl:*) + # --allow-tool shell(ci-evidence-reader:*) # --allow-tool shell(cut) # --allow-tool shell(date) # --allow-tool shell(dirname) @@ -856,7 +859,7 @@ jobs: fi # shellcheck disable=SC1003,SC2016,SC2086 awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(awk)'\'' --allow-tool '\''shell(basename)'\'' --allow-tool '\''shell(bash)'\'' --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(chmod)'\'' --allow-tool '\''shell(curl:*)'\'' --allow-tool '\''shell(cut)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(dirname)'\'' --allow-tool '\''shell(dotnet:*)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(env)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(git:*)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(mkdir)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sed)'\'' --allow-tool '\''shell(sh)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(tee)'\'' --allow-tool '\''shell(test)'\'' --allow-tool '\''shell(tr)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(xargs)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(awk)'\'' --allow-tool '\''shell(basename)'\'' --allow-tool '\''shell(bash)'\'' --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(chmod)'\'' --allow-tool '\''shell(ci-evidence-reader:*)'\'' --allow-tool '\''shell(cut)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(dirname)'\'' --allow-tool '\''shell(dotnet:*)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(env)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(git:*)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(mkdir)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sed)'\'' --allow-tool '\''shell(sh)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(tee)'\'' --allow-tool '\''shell(test)'\'' --allow-tool '\''shell(tr)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(xargs)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -1246,7 +1249,7 @@ jobs: GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} GH_AW_WORKFLOW_ID: "ci-failure-scan" - GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "0" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" GH_AW_ENGINE_ID: "copilot" GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 3cec38e17e1dbd..edcc3dde445981 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -44,7 +44,7 @@ tools: toolsets: [pull_requests, repos, issues, search] min-integrity: approved edit: - bash: ["dotnet", "git", "find", "ls", "cat", "grep", "head", "tail", "wc", "curl", "jq", "tee", "sed", "awk", "tr", "cut", "sort", "uniq", "xargs", "echo", "date", "mkdir", "test", "env", "basename", "dirname", "bash", "sh", "chmod"] + bash: ["dotnet", "git", "find", "ls", "cat", "grep", "head", "tail", "wc", "jq", "tee", "sed", "awk", "tr", "cut", "sort", "uniq", "xargs", "echo", "date", "mkdir", "test", "env", "basename", "dirname", "bash", "sh", "chmod", "ci-evidence-reader:*"] checkout: fetch-depth: 50 @@ -64,6 +64,14 @@ network: - dev.azure.com - helix.dot.net - "*.blob.core.windows.net" + +pre-agent-steps: + - name: Install CI evidence reader + run: | + mkdir -p "${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin" + cp .github/workflows/ci-evidence-reader "${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin/ci-evidence-reader" + chmod 0555 "${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin/ci-evidence-reader" + printf '%s\n' "${RUNNER_TEMP}/gh-aw/ci-evidence-tools/bin" >> "$GITHUB_PATH" --- # CI Outer-Loop Failure Scanner @@ -128,7 +136,7 @@ Read once at start: For each row in the pipeline table below: -1. Pre-bind the build-list URL to a shell variable, then `curl -s "$url" | tee /tmp/gh-aw/agent/builds_.json`. Fetch at least 25 builds. +1. Run `ci-evidence-reader azdo-builds --definition --top 25 --skip 0 --output /tmp/gh-aw/agent/builds_.json`. Fetch at least 25 builds. 2. Pick `source` = most recent build with `result in {failed, partiallySucceeded}` that has at least one COMPLETED build with a strictly later `finishTime`. That later build is the `follow_up` anchor for Step 3.5; without it, a freshly-fixed regression cannot be distinguished from a still-failing one. (The dnceng-public build-list is sorted DESC by `queueTime`, so `source` will appear AFTER its `follow_up` in the JSON array; "later in time" refers to wall-clock, not array position.) 3. Skip reasons: `source.finishTime > 14d` -> `pipeline-skipped: stale build window (>14d)`. No `follow_up` (source is the absolute latest) -> `pipeline-skipped: no follow-up build yet — defer to next run`. No qualifying build in 7 days -> `pipeline-skipped: stale`. The 14-day window accommodates JIT-stress family pipelines (defs 109–160, 230, 235) that run on a weekly-or-longer cadence; tightening to 72h blanket-suppresses their actionable failures. 4. Otherwise pass `source`'s failed timeline records to Step 3. @@ -200,8 +208,8 @@ Do **not** collapse a build to one arbitrarily chosen failed `Send to Helix` log Save the canonical failure log to `/tmp/gh-aw/agent/failure.log` per signature before extracting; KBE check 7 greps it for the verbatim signature. ```bash -log_url="" -curl -s "$log_url" | tee /tmp/gh-aw/agent/failure.log | tail -5 +ci-evidence-reader helix-console --job-id JOBID --work-item "WORKITEM" --output /tmp/gh-aw/agent/failure.log +tail -5 /tmp/gh-aw/agent/failure.log ``` 1. **Build break.** Failed task is `Build product` / `Build native components` / `Configure CMake` / any pre-test compile step, AND `Send to Helix` is `skipped`. Read the signature from the failing compile task log (CSxxxx / linker error / cmake error line). @@ -217,9 +225,10 @@ If the same signature appears in *every* sampled build (100% failure rate in the #### Data sources - **AzDO REST.** `https://dev.azure.com/dnceng-public/public/_apis/build/...`. Anonymous, no auth. - - List builds: `?definitions={id}&branchName=refs/heads/main&statusFilter=completed&resultFilter=succeeded,failed,partiallySucceeded&%24top=25&api-version=7.1` - - Timeline: `/builds/{id}/timeline?api-version=7.1` returns flat `records[]`; reconstruct via `parentId`. A failed record with non-null log id is a leaf to inspect. -- **Helix REST.** `https://helix.dot.net/api/jobs/{jobId}/workitems?api-version=2019-06-17`. Each item has `Name`, `State`, `ExitCode`, `ConsoleOutputUri`. Failed: `ExitCode != 0` or `State == "Failed"`. + - List builds: `ci-evidence-reader azdo-builds --definition ID --top 25 --skip N --output PATH.json`, with `N` restricted to `0,10,20,30,40`. + - Timeline: `ci-evidence-reader azdo-timeline --build-id ID --output PATH.json` returns flat `records[]`; reconstruct via `parentId`. A failed record with non-null log id is a leaf to inspect. + - Task log: `ci-evidence-reader azdo-log --build-id ID --log-id ID --output PATH.log`. +- **Helix REST.** `ci-evidence-reader helix-work-items --job-id JOBID --output PATH.json` returns items with `Name`, `State`, `ExitCode`, and `ConsoleOutputUri`. Failed: `ExitCode != 0` or `State == "Failed"`. Use `ci-evidence-reader helix-console --job-id JOBID --work-item "WORKITEM" --output PATH.log` for the selected console. - **Build Analysis GitHub check (best-effort).** Read the source SHA from the AzDO build, then query `GET /repos/dotnet/runtime/commits/{sha}/check-runs` and inspect the completed @@ -389,21 +398,15 @@ Sanitize every log excerpt in KBE issue bodies using ## Environment constraints -These look like permission errors but are physical. - -- **Pre-bind every URL to a shell variable on its own line, then `curl -s "$url"`.** Inline URLs with `?` or `&` are rejected as "Permission denied" even single-quoted (the tool-approver treats query strings as interactive prompts). Working pattern: - - ```bash - url='https://dev.azure.com/dnceng-public/public/_apis/build/builds?definitions=154&branchName=refs/heads/main&statusFilter=completed&resultFilter=succeeded,failed,partiallySucceeded&%24top=25&api-version=7.1' - curl -s "$url" | jq '.' | tee /tmp/gh-aw/agent/builds.json | jq -r '.value[0] | "\(.id) \(.result)"' - ``` - - Do NOT retry an inline URL hoping the rejection clears. Switch to the variable pattern immediately. +All AzDO and Helix reads must use the repository-owned +`ci-evidence-reader` helper installed by `pre-agent-steps`. It constructs and +validates the fixed HTTPS endpoints, permits only GET requests, follows only +allow-listed redirects, enforces response limits and a 30-second timeout, and writes +only below `/tmp/gh-aw/agent/`. Use `--top 25` and `--skip 0|10|20|30|40` for build +pagination. Do not invoke another HTTP client, construct URLs in shell, or use +`actions/setup-python`; the execution image must provide `python3`. -- **No `>` or `-o` redirection.** Use `| tee /path/to/file`. -- **No `$(...)` or `${var@P}`.** Compose via `xargs -I{}` or by reading files inline. -- **OData `$top` must be encoded as `%24top` in URLs.** -- **Bash allowlist** (per the frontmatter `tools.bash`): `dotnet`, `git`, `find`, `ls`, `cat`, `grep`, `head`, `tail`, `wc`, `curl`, `jq`, `tee`, `sed`, `awk`, `tr`, `cut`, `sort`, `uniq`, `xargs`, `echo`, `date`, `mkdir`, `test`, `env`, `basename`, `dirname`, `bash`, `sh`, `chmod`. No `gh`, no `pwsh`, no `python`. +- **Bash allowlist** (per the frontmatter `tools.bash`): `dotnet`, `git`, `find`, `ls`, `cat`, `grep`, `head`, `tail`, `wc`, `jq`, `tee`, `sed`, `awk`, `tr`, `cut`, `sort`, `uniq`, `xargs`, `echo`, `date`, `mkdir`, `test`, `env`, `basename`, `dirname`, `bash`, `sh`, `chmod`, `ci-evidence-reader:*`. No `gh`, `pwsh`, or `python`. - **Each bash call runs in a fresh subshell.** Persist state to `/tmp/gh-aw/agent/`. ## Output discipline diff --git a/.github/workflows/tests/test_ci_evidence_reader.py b/.github/workflows/tests/test_ci_evidence_reader.py new file mode 100644 index 00000000000000..99e0cb23104386 --- /dev/null +++ b/.github/workflows/tests/test_ci_evidence_reader.py @@ -0,0 +1,263 @@ +import importlib.machinery +import importlib.util +import io +import os +import sys +import tempfile +import unittest +import urllib.error +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "ci-evidence-reader" +LOADER = importlib.machinery.SourceFileLoader("ci_evidence_reader", str(SCRIPT)) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +ci_evidence_reader = importlib.util.module_from_spec(SPEC) +LOADER.exec_module(ci_evidence_reader) + + +class FakeResponse: + def __init__(self, data=b"ok", status=200, content_length=None): + self._stream = io.BytesIO(data) + self.status = status + self.headers = {} + if content_length is not None: + self.headers["Content-Length"] = str(content_length) + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def read(self, size): + return self._stream.read(size) + + +class FakeOpener: + def __init__(self, response=None, error=None): + self.response = response + self.error = error + self.requests = [] + + def open(self, request, timeout): + self.requests.append((request, timeout)) + if self.error: + raise self.error + return self.response + + +class UrlValidationTests(unittest.TestCase): + def test_accepts_reader_azdo_build_list(self): + url = ci_evidence_reader._azdo_builds_url(154, 25, 20) + ci_evidence_reader._validate_url(url, {"azdo"}) + self.assertIn( + "definitions=154&branchName=refs/heads/main&statusFilter=completed" + "&resultFilter=succeeded,failed,partiallySucceeded&%24top=25&%24skip=20" + "&api-version=7.1", + url, + ) + + def test_rejects_unlisted_definition(self): + url = ci_evidence_reader._azdo_builds_url(999) + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._validate_url(url, {"azdo"}) + + def test_definition_allowlist_matches_workflow_table(self): + workflow = Path(__file__).parents[1] / "ci-failure-scan.md" + expected = { + int(line.split("|")[2].strip()) + for line in workflow.read_text(encoding="utf-8").splitlines() + if line.startswith("| ") and line.count("|") >= 3 + and line.split("|")[2].strip().isdigit() + } + self.assertEqual(tuple(sorted(expected)), ci_evidence_reader.DEFINITION_IDS) + + def test_rejects_invalid_top_and_skip(self): + for top in (1, 24, 26, 100): + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._validate_url( + ci_evidence_reader._azdo_builds_url(154, top), {"azdo"} + ) + for skip in (1, 9, 50): + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._validate_url( + ci_evidence_reader._azdo_builds_url(154, 25, skip), {"azdo"} + ) + + def test_rejects_credentials_and_arbitrary_hosts(self): + for url in ( + "https://user:password@dev.azure.com/dnceng-public/public/_apis/build/builds", + "https://example.com/", + ): + with self.subTest(url=url): + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._validate_url(url, {"azdo"}) + + def test_accepts_only_helix_console_blobs(self): + valid = ( + "https://helixre107v0xdeko0k025g8.blob.core.windows.net/" + "dotnet-runtime-refs-heads-main/job/1/console.1234.log" + "?sv=2020-01-01&sr=c&sig=signature&se=2030-01-01&sp=rl" + ) + ci_evidence_reader._validate_url(valid, {"helix-console"}) + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._validate_url( + "https://other.blob.core.windows.net/container/secrets.txt", + {"helix-console"}, + ) + + def test_rejects_undocumented_blob_query_parameter(self): + url = ( + "https://helixre107v0xdeko0k025g8.blob.core.windows.net/" + "dotnet-runtime-refs-heads-main/job/1/console.1234.log?sk=value" + ) + with self.assertRaisesRegex( + ci_evidence_reader.TransportError, "unexpected Helix console blob" + ): + ci_evidence_reader._validate_url(url, {"helix-console"}) + + def test_helix_work_items_use_specific_family(self): + url = ci_evidence_reader._helix_work_items_url( + "00000000-0000-0000-0000-000000000000" + ) + ci_evidence_reader._validate_url(url, {"helix-work-items"}) + + def test_redirect_handler_rejects_family_escape(self): + handler = ci_evidence_reader._ValidatingRedirectHandler({"azdo"}) + with self.assertRaises(ci_evidence_reader.TransportError): + handler.redirect_request( + None, + None, + 302, + "Found", + {}, + "https://helix.dot.net/api/jobs/" + "00000000-0000-0000-0000-000000000000/workitems" + "?api-version=2019-06-17", + ) + + +class OutputPathTests(unittest.TestCase): + def setUp(self): + self.original_root = ci_evidence_reader.OUTPUT_ROOT + self.temp = tempfile.TemporaryDirectory() + ci_evidence_reader.OUTPUT_ROOT = Path(self.temp.name) + + def tearDown(self): + ci_evidence_reader.OUTPUT_ROOT = self.original_root + self.temp.cleanup() + + def test_accepts_output_below_root(self): + output = Path(self.temp.name) / "metadata" / "builds.json" + self.assertEqual( + ci_evidence_reader._validate_output_path(str(output), (".json",)), + output.resolve(), + ) + + def test_rejects_output_outside_root_and_wrong_suffix(self): + cases = ( + (str(Path(self.temp.name).parent / "outside.json"), (".json",)), + (str(Path(self.temp.name) / "file.tsv"), (".json",)), + ) + for output, suffixes in cases: + with self.subTest(output=output): + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._validate_output_path(output, suffixes) + + def test_rejects_existing_directory(self): + output = Path(self.temp.name) / "directory.json" + output.mkdir() + with self.assertRaisesRegex( + ci_evidence_reader.TransportError, "regular file" + ): + ci_evidence_reader._validate_output_path(str(output), (".json",)) + + def test_rejects_symlink_parent_outside_root(self): + symlink_parent = Path(self.temp.name) / "symlink-parent" + symlink_parent.symlink_to( + Path(self.temp.name).parent, target_is_directory=True + ) + + with self.assertRaisesRegex( + ci_evidence_reader.TransportError, "output path must be under" + ): + ci_evidence_reader._validate_output_path( + str(symlink_parent / "output.json"), (".json",) + ) + + def test_rejects_symlink_parent_during_write(self): + real_parent = Path(self.temp.name) / "real-parent" + real_parent.mkdir() + symlink_parent = Path(self.temp.name) / "symlink-parent" + symlink_parent.symlink_to(real_parent, target_is_directory=True) + + with self.assertRaisesRegex( + ci_evidence_reader.TransportError, "real directory" + ): + ci_evidence_reader._write_output( + b"data", str(symlink_parent / "output.json"), (".json",) + ) + + +class RequestBehaviorTests(unittest.TestCase): + def setUp(self): + self.url = ci_evidence_reader._azdo_builds_url(154) + + def test_get_only_with_fixed_timeout_and_user_agent(self): + opener = FakeOpener(FakeResponse(b"{}")) + self.assertEqual( + ci_evidence_reader._request_bytes( + self.url, {"azdo"}, ci_evidence_reader.JSON_LIMIT, opener + ), + b"{}", + ) + request, timeout = opener.requests[0] + self.assertEqual(request.get_method(), "GET") + self.assertEqual(timeout, ci_evidence_reader.TIMEOUT_SECONDS) + self.assertEqual(request.get_header("User-agent"), ci_evidence_reader.USER_AGENT) + + def test_rejects_content_length_and_stream_over_limit(self): + for response in ( + FakeResponse(b"small", content_length=11), + FakeResponse(b"01234567890"), + ): + with self.subTest(response=response): + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._request_bytes( + self.url, {"azdo"}, 10, FakeOpener(response) + ) + + def test_surfaces_http_errors(self): + error = urllib.error.HTTPError(self.url, 503, "Unavailable", {}, None) + with self.assertRaisesRegex(ci_evidence_reader.TransportError, "status 503"): + ci_evidence_reader._request_bytes( + self.url, {"azdo"}, 10, FakeOpener(error=error) + ) + + +class HelixTraversalTests(unittest.TestCase): + def test_console_url_is_selected_by_exact_work_item_name(self): + payload = b"""[ + { + "Name": "runtime-tests", + "ConsoleOutputUri": "https://helixre107v0xdeko0k025g8.blob.core.windows.net/dotnet-runtime/job/console.1.log?helixlogtype=result" + } + ]""" + self.assertIn( + "console.1.log", + ci_evidence_reader._console_url(payload, "runtime-tests"), + ) + + def test_console_url_rejects_untrusted_metadata(self): + payload = b"""[ + { + "Name": "runtime-tests", + "ConsoleOutputUri": "https://example.com/console.log" + } + ]""" + with self.assertRaises(ci_evidence_reader.TransportError): + ci_evidence_reader._console_url(payload, "runtime-tests") + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From c2a6cfb4cb21bd6c4826b8c719728171c9cfb509 Mon Sep 17 00:00:00 2001 From: vitek-karas <10670590+vitek-karas@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:20:02 +0200 Subject: [PATCH 2/3] Allow CI scan definition IDs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 65c2a187-ce76-4264-9d3e-ee20fc9e7565 --- .github/workflows/ci-evidence-reader | 56 ++----------------- .../tests/test_ci_evidence_reader.py | 15 +---- 2 files changed, 6 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci-evidence-reader b/.github/workflows/ci-evidence-reader index e112197bdbb7c7..a474946b537c9c 100644 --- a/.github/workflows/ci-evidence-reader +++ b/.github/workflows/ci-evidence-reader @@ -19,44 +19,6 @@ TIMEOUT_SECONDS = 30 JSON_LIMIT = 16 * 1024 * 1024 LOG_LIMIT = 64 * 1024 * 1024 CHUNK_SIZE = 64 * 1024 -DEFINITION_IDS = ( - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 123, - 124, - 134, - 136, - 137, - 138, - 140, - 141, - 144, - 145, - 146, - 150, - 153, - 154, - 155, - 159, - 160, - 230, - 235, - 265, - 309, - 316, - 330, -) ALLOWED_SKIPS = (0, 10, 20, 30, 40) AZDO_HOST = "dev.azure.com" AZDO_PATH_PREFIX = "/dnceng-public/public/_apis/build/" @@ -161,17 +123,16 @@ def _validate_azdo_url(parts: urllib.parse.SplitResult, query: dict[str, str]) - if set(query) != expected_keys: raise TransportError("unexpected Azure DevOps build-list query parameters") try: - definition = int(query["definitions"]) + definition = _positive_int(query["definitions"]) top = int(query["$top"]) - except ValueError as error: + except (ValueError, argparse.ArgumentTypeError) as error: raise TransportError("invalid Azure DevOps build-list query") from error try: skip = int(query["$skip"]) except ValueError as error: raise TransportError("invalid Azure DevOps build-list query") from error if ( - definition not in DEFINITION_IDS - or top != 25 + top != 25 or skip not in ALLOWED_SKIPS ): raise TransportError("Azure DevOps build-list query is outside permitted bounds") @@ -449,15 +410,6 @@ def _positive_int(value: str) -> int: return parsed -def _definition_id(value: str) -> int: - parsed = _positive_int(value) - if parsed not in DEFINITION_IDS: - raise argparse.ArgumentTypeError( - f"must be one of: {', '.join(str(item) for item in DEFINITION_IDS)}" - ) - return parsed - - def _top(value: str) -> int: try: parsed = int(value) @@ -614,7 +566,7 @@ def _parser() -> argparse.ArgumentParser: subparsers = parser.add_subparsers(dest="command", required=True) builds = subparsers.add_parser("azdo-builds") - builds.add_argument("--definition", required=True, type=_definition_id) + builds.add_argument("--definition", required=True, type=_positive_int) builds.add_argument("--top", type=_top, default=25) builds.add_argument("--skip", type=_skip, default=0) builds.add_argument("--output", required=True) diff --git a/.github/workflows/tests/test_ci_evidence_reader.py b/.github/workflows/tests/test_ci_evidence_reader.py index 99e0cb23104386..c0448a542c06fd 100644 --- a/.github/workflows/tests/test_ci_evidence_reader.py +++ b/.github/workflows/tests/test_ci_evidence_reader.py @@ -58,20 +58,9 @@ def test_accepts_reader_azdo_build_list(self): url, ) - def test_rejects_unlisted_definition(self): + def test_accepts_any_positive_definition(self): url = ci_evidence_reader._azdo_builds_url(999) - with self.assertRaises(ci_evidence_reader.TransportError): - ci_evidence_reader._validate_url(url, {"azdo"}) - - def test_definition_allowlist_matches_workflow_table(self): - workflow = Path(__file__).parents[1] / "ci-failure-scan.md" - expected = { - int(line.split("|")[2].strip()) - for line in workflow.read_text(encoding="utf-8").splitlines() - if line.startswith("| ") and line.count("|") >= 3 - and line.split("|")[2].strip().isdigit() - } - self.assertEqual(tuple(sorted(expected)), ci_evidence_reader.DEFINITION_IDS) + ci_evidence_reader._validate_url(url, {"azdo"}) def test_rejects_invalid_top_and_skip(self): for top in (1, 24, 26, 100): From fcc3c838624e767d70070ba4db5b23a45d22129b Mon Sep 17 00:00:00 2001 From: vitek-karas <10670590+vitek-karas@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:36:24 +0200 Subject: [PATCH 3/3] Simplify CI evidence requests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 65c2a187-ce76-4264-9d3e-ee20fc9e7565 --- .github/workflows/ci-evidence-reader | 108 ++++++++++------ .../tests/test_ci_evidence_reader.py | 118 ++++++++++++++++++ 2 files changed, 190 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci-evidence-reader b/.github/workflows/ci-evidence-reader index a474946b537c9c..f06086f1f98bbf 100644 --- a/.github/workflows/ci-evidence-reader +++ b/.github/workflows/ci-evidence-reader @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +from dataclasses import dataclass import json import os import re @@ -12,6 +13,7 @@ import urllib.parse import urllib.request import uuid from pathlib import Path +from typing import Callable OUTPUT_ROOT = Path("/tmp/gh-aw/agent") @@ -466,6 +468,68 @@ def _helix_work_items_url(job_id: str) -> str: ) +def _azdo_timeline_url(build_id: int) -> str: + return ( + f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds/{build_id}/timeline" + "?api-version=7.1" + ) + + +def _azdo_log_url(build_id: int, log_id: int) -> str: + return ( + f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds/{build_id}/logs/{log_id}" + "?api-version=7.1" + ) + + +@dataclass(frozen=True) +class _RequestSpec: + argument_names: tuple[str, ...] + url_builder: Callable[..., str] + allowed_families: set[str] + limit: int + suffixes: tuple[str, ...] + + +_REQUEST_SPECS = { + "azdo-builds": _RequestSpec( + ("definition", "top", "skip"), + _azdo_builds_url, + {"azdo"}, + JSON_LIMIT, + (".json",), + ), + "azdo-timeline": _RequestSpec( + ("build_id",), + _azdo_timeline_url, + {"azdo"}, + JSON_LIMIT, + (".json",), + ), + "azdo-log": _RequestSpec( + ("build_id", "log_id"), + _azdo_log_url, + {"azdo"}, + LOG_LIMIT, + (".log", ".txt"), + ), + "helix-work-items": _RequestSpec( + ("job_id",), + _helix_work_items_url, + {"helix-work-items"}, + JSON_LIMIT, + (".json",), + ), +} + + +def _request_from_spec(args: argparse.Namespace, spec: _RequestSpec) -> bytes: + arguments = (getattr(args, name) for name in spec.argument_names) + return _request_bytes( + spec.url_builder(*arguments), spec.allowed_families, spec.limit + ) + + def _json_items(payload: bytes) -> list[dict]: try: document = json.loads(payload) @@ -519,44 +583,16 @@ def _console_url(payload: bytes, work_item: str) -> str: def _run(args: argparse.Namespace) -> None: - if args.command == "azdo-builds": - data = _request_bytes( - _azdo_builds_url(args.definition, args.top, args.skip), {"azdo"}, JSON_LIMIT - ) - _write_output(data, args.output, (".json",)) - elif args.command == "azdo-timeline": - url = ( - f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds/{args.build_id}/timeline" - "?api-version=7.1" - ) - _write_output( - _request_bytes(url, {"azdo"}, JSON_LIMIT), args.output, (".json",) - ) - elif args.command == "azdo-log": - url = ( - f"https://{AZDO_HOST}{AZDO_PATH_PREFIX}builds/{args.build_id}/logs/" - f"{args.log_id}?api-version=7.1" - ) - _write_output( - _request_bytes(url, {"azdo"}, LOG_LIMIT), args.output, (".log", ".txt") - ) - elif args.command == "helix-work-items": - data = _request_bytes( - _helix_work_items_url(args.job_id), {"helix-work-items"}, JSON_LIMIT - ) - _write_output(data, args.output, (".json",)) - elif args.command == "helix-console": - work_items = _request_bytes( - _helix_work_items_url(args.job_id), {"helix-work-items"}, JSON_LIMIT - ) + if args.command == "helix-console": + work_items = _request_from_spec(args, _REQUEST_SPECS["helix-work-items"]) console_url = _console_url(work_items, args.work_item) - _write_output( - _request_bytes(console_url, {"helix-console"}, LOG_LIMIT), - args.output, - (".log", ".txt"), - ) + data = _request_bytes(console_url, {"helix-console"}, LOG_LIMIT) + suffixes = (".log", ".txt") else: - raise AssertionError(f"unhandled command: {args.command}") + spec = _REQUEST_SPECS[args.command] + data = _request_from_spec(args, spec) + suffixes = spec.suffixes + _write_output(data, args.output, suffixes) def _parser() -> argparse.ArgumentParser: diff --git a/.github/workflows/tests/test_ci_evidence_reader.py b/.github/workflows/tests/test_ci_evidence_reader.py index c0448a542c06fd..14f048124326cc 100644 --- a/.github/workflows/tests/test_ci_evidence_reader.py +++ b/.github/workflows/tests/test_ci_evidence_reader.py @@ -7,6 +7,7 @@ import unittest import urllib.error from pathlib import Path +from unittest import mock SCRIPT = Path(__file__).parents[1] / "ci-evidence-reader" @@ -225,6 +226,123 @@ def test_surfaces_http_errors(self): ) +class CommandDispatchTests(unittest.TestCase): + def test_direct_commands_use_their_request_specs(self): + cases = ( + ( + ["azdo-builds", "--definition", "999", "--output", "/tmp/builds.json"], + ci_evidence_reader._azdo_builds_url(999), + {"azdo"}, + ci_evidence_reader.JSON_LIMIT, + (".json",), + ), + ( + [ + "azdo-timeline", + "--build-id", + "123", + "--output", + "/tmp/timeline.json", + ], + ci_evidence_reader._azdo_timeline_url(123), + {"azdo"}, + ci_evidence_reader.JSON_LIMIT, + (".json",), + ), + ( + [ + "azdo-log", + "--build-id", + "123", + "--log-id", + "456", + "--output", + "/tmp/log.log", + ], + ci_evidence_reader._azdo_log_url(123, 456), + {"azdo"}, + ci_evidence_reader.LOG_LIMIT, + (".log", ".txt"), + ), + ( + [ + "helix-work-items", + "--job-id", + "00000000-0000-0000-0000-000000000000", + "--output", + "/tmp/work-items.json", + ], + ci_evidence_reader._helix_work_items_url( + "00000000-0000-0000-0000-000000000000" + ), + {"helix-work-items"}, + ci_evidence_reader.JSON_LIMIT, + (".json",), + ), + ) + + for command, url, families, limit, suffixes in cases: + with self.subTest(command=command[0]): + args = ci_evidence_reader._parser().parse_args(command) + with ( + mock.patch.object( + ci_evidence_reader, "_request_bytes", return_value=b"payload" + ) as request, + mock.patch.object(ci_evidence_reader, "_write_output") as write, + ): + ci_evidence_reader._run(args) + request.assert_called_once_with(url, families, limit) + write.assert_called_once_with(b"payload", args.output, suffixes) + + def test_helix_console_resolves_then_reads_console(self): + job_id = "00000000-0000-0000-0000-000000000000" + args = ci_evidence_reader._parser().parse_args( + [ + "helix-console", + "--job-id", + job_id, + "--work-item", + "runtime-tests", + "--output", + "/tmp/console.log", + ] + ) + console_url = ( + "https://helixre107v0xdeko0k025g8.blob.core.windows.net/" + "dotnet-runtime/job/console.1.log?helixlogtype=result" + ) + with ( + mock.patch.object( + ci_evidence_reader, + "_request_bytes", + side_effect=[b"work-items", b"console"], + ) as request, + mock.patch.object( + ci_evidence_reader, "_console_url", return_value=console_url + ) as resolve_console, + mock.patch.object(ci_evidence_reader, "_write_output") as write, + ): + ci_evidence_reader._run(args) + + request.assert_has_calls( + [ + mock.call( + ci_evidence_reader._helix_work_items_url(job_id), + {"helix-work-items"}, + ci_evidence_reader.JSON_LIMIT, + ), + mock.call( + console_url, {"helix-console"}, ci_evidence_reader.LOG_LIMIT + ), + ] + ) + self.assertEqual(request.call_count, 2) + resolve_console.assert_called_once_with(b"work-items", "runtime-tests") + write.assert_called_once_with( + b"console", args.output, (".log", ".txt") + ) + + class HelixTraversalTests(unittest.TestCase): def test_console_url_is_selected_by_exact_work_item_name(self): payload = b"""[