From 6ec07fcef8ec86183029dc5c5758e6c9354effe1 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 13:43:01 +0100 Subject: [PATCH 01/11] Add the Bits review merge gate Gate: only the trusted Bits publisher app's PASS on the exact open-main head counts, shared as a conjunction across PRs sharing a head. Fail closed on edited or unknown-bot reviews; never fall back to other tokens. Trusted publisher and allowlist live in the Chainguard policy; the event receiver fans the publisher per distinct resolved head. --- ...f.bits-review-gate.publish-status.sts.yaml | 12 + .github/scripts/bits_review_gate.py | 864 ++++++++++++++++++ .github/scripts/bits_review_gate_policy.py | 446 +++++++++ .github/workflows/bits-review-event.yml | 17 + .github/workflows/bits-review-gate.yml | 82 ++ 5 files changed, 1421 insertions(+) create mode 100644 .github/chainguard/self.bits-review-gate.publish-status.sts.yaml create mode 100644 .github/scripts/bits_review_gate.py create mode 100644 .github/scripts/bits_review_gate_policy.py create mode 100644 .github/workflows/bits-review-event.yml create mode 100644 .github/workflows/bits-review-gate.yml diff --git a/.github/chainguard/self.bits-review-gate.publish-status.sts.yaml b/.github/chainguard/self.bits-review-gate.publish-status.sts.yaml new file mode 100644 index 0000000000..28bc4e1b16 --- /dev/null +++ b/.github/chainguard/self.bits-review-gate.publish-status.sts.yaml @@ -0,0 +1,12 @@ +issuer: https://token.actions.githubusercontent.com + +subject_pattern: repo:DataDog/java-profiler:(pull_request|ref:refs/heads/main) + +claim_pattern: + event_name: (pull_request_target|workflow_run|schedule|workflow_dispatch) + ref: refs/heads/main + ref_protected: "true" + job_workflow_ref: DataDog/java-profiler/\.github/workflows/bits-review-gate\.yml@refs/heads/main + +permissions: + statuses: write diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py new file mode 100644 index 0000000000..fc12a70938 --- /dev/null +++ b/.github/scripts/bits_review_gate.py @@ -0,0 +1,864 @@ +# Copyright 2026, Datadog, Inc. +"""GitHub API layer and CLI for the Bits review merge gate. + +``GitHubClient`` reads pull requests, PR reviews and commit statuses +through an injectable transport (the stdlib ``urllib.request`` by +default). Every read fails closed through :class:`ApiError`: transport +failures, non-2xx after retries, GraphQL ``errors`` (even with partial +data), malformed JSON and missing response structure never yield a +partial list. Reads carry ``GITHUB_TOKEN`` only; ``STATUS_TOKEN`` is +never read here. + +On top of the client, ``normalize_review`` maps GraphQL review nodes to +the policy's record shape and ``read_sha_snapshot`` wires the policy +validators into the fetch path, recording api-error and invalid-input +flags instead of skipping or truncating malformed input. +``revalidated_decision`` composes the decision on that seam: it reads one +snapshot and computes the decision with the policy — return values only: +no status is read or written there. The ``publish --sha H`` +subcommand consumes the seam's decision: it suppresses the write when +the newest ``bits-review/head`` status created by ``dd-octo-sts[bot]`` +already equals the decision, else POSTs it with ``STATUS_TOKEN`` only; +an unchanged or published ``error`` decision still exits 1. The +``resolve`` subcommand turns event selectors, received only through +environment +variables, into the deduplicated head-SHA list to publish; it never +interpolates event text into a command and executes nothing. +""" + +import argparse +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +import bits_review_gate_policy as policy + +#: Total attempts for network errors, 5xx responses and rate-limited +#: 403/429 responses; every other 4xx gets one. +MAX_ATTEMPTS = 3 +_RETRY_BACKOFF_SECONDS = 0.2 +_TIMEOUT_SECONDS = 30 +REST_PER_PAGE = 100 +REVIEW_PAGE_SIZE = 100 +#: Distinct head SHAs one resolve run may emit; more is an error, never a +#: truncation. +MAX_RESOLVE_SHAS = 256 +#: Pages one paginated read (GraphQL reviews, REST lists) may follow; more +#: is an error, never a truncation. +MAX_FETCH_PAGES = 20 +#: Upper bound for honoring a server-provided rate-limit window +#: (Retry-After, else x-ratelimit-reset), so a primary limit cannot stall +#: the workflow run. +_MAX_RETRY_AFTER_SECONDS = 300 +_SHA_RE = re.compile(r"[0-9a-f]{40}") +_DEFAULT_API_URL = "https://api.github.com" +_DEFAULT_GRAPHQL_URL = "https://api.github.com/graphql" +#: The app login whose statuses are this gate's own output; only its +#: entries are candidates for unchanged-status suppression. +STATUS_CREATOR = "dd-octo-sts[bot]" + +#: The contract's review query: identity, state, body, timestamps, the +#: head commit and author typing. No edit-history field is requested. +_REVIEWS_QUERY = """query($owner: String!, $name: String!, $number: Int!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviews(first: %d, after: $after) { + pageInfo { hasNextPage endCursor } + nodes { + databaseId + state + body + submittedAt + lastEditedAt + commit { oid } + author { + __typename + login + ... on Bot { databaseId } + ... on User { databaseId } + } + } + } + } + } +} +""" % REVIEW_PAGE_SIZE + + +class ApiError(Exception): + """The fail-closed signal every failed or malformed API read raises.""" + + +class RateLimitError(ApiError): + """An API read that failed because the rate-limit window exceeds what + honoring retries can cover.""" + + +class ResolveError(Exception): + """Invalid resolve input, or a result the resolver refuses to truncate.""" + + +def _urllib_transport(request): + """The default transport: one stdlib urllib call per request.""" + return urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) + + +def _next_link(link_header, base_url): + """The rel="next" URL of a Link header constrained to the API origin + (same scheme and host), or None on the last page.""" + if not link_header: + return None + base = urllib.parse.urlsplit(base_url) + for next_url in re.findall(r'<([^>]*)>\s*;\s*rel="next"', link_header): + if not next_url: + continue + parts = urllib.parse.urlsplit(next_url) + if (parts.scheme.lower(), parts.netloc.lower()) != (base.scheme.lower(), base.netloc.lower()): + raise ApiError("cross-origin next-page URL: %r" % (next_url,)) + return next_url + return None + + +def _header_lookup(headers, name): + """The value of one header matched case-insensitively, or None: + urllib exposes ``email.message.Message`` while test fakes may carry + plain case-sensitive dicts.""" + if not headers: + return None + lowered = name.lower() + for key, value in headers.items(): + if key.lower() == lowered: + return value + return None + + +def _is_rate_limited(status, headers): + """True for a 403/429 GitHub rate-limit response. + + A 429 is rate-limited only when a Retry-After header is present — + remaining 0 alone never retries it. A 403 is rate-limited only when + ``x-ratelimit-remaining`` is 0; a plain 403 is never retried, with + or without Retry-After. + """ + if status not in (403, 429): + return False + remaining = _header_lookup(headers, "x-ratelimit-remaining") + if status == 403: + return remaining is not None and str(remaining).strip() == "0" + return _header_lookup(headers, "Retry-After") is not None + + +def _retry_after_seconds(headers): + """The integer-seconds Retry-After delay of the headers, or None when + absent or unparseable (HTTP-date form is never parsed). The delay is + never negative: a hostile value clamps to zero.""" + value = _header_lookup(headers, "Retry-After") + if value is None: + return None + try: + return max(0, int(str(value).strip())) + except ValueError: + return None + + +def _rate_limit_window_seconds(headers): + """The rate-limit window the headers advertise, in seconds, or None + when neither signal parses: the Retry-After delay if present (already + non-negative), else the seconds until the ``x-ratelimit-reset`` epoch, + clamped to zero. An HTTP-date Retry-After is never parsed, so it + falls through to the reset reading.""" + retry_after = _retry_after_seconds(headers) + if retry_after is not None: + return retry_after + reset = _header_lookup(headers, "x-ratelimit-reset") + if reset is None: + return None + try: + return max(0, int(str(reset).strip()) - time.time()) + except ValueError: + return None + + +def _close_quietly(response): + """Close one response-like object, tolerating one without ``close``.""" + close = getattr(response, "close", None) + if close is not None: + close() + + +class GitHubClient: + """Reads the gate's decision inputs from the GitHub API. + + ``transport`` is a callable taking a ``urllib.request.Request`` and + returning an HTTP-response-like object (``status``, ``read()``, + ``headers``). ``env`` supplies ``GITHUB_API_URL``, + ``GITHUB_GRAPHQL_URL`` and ``GITHUB_TOKEN``; ``repo`` is the + ``owner/name`` repository. + """ + + def __init__(self, repo, transport=None, env=None): + self._repo = repo + self._transport = _urllib_transport if transport is None else transport + env = os.environ if env is None else env + self._api_url = env.get("GITHUB_API_URL", _DEFAULT_API_URL).rstrip("/") + self._graphql_url = env.get("GITHUB_GRAPHQL_URL", _DEFAULT_GRAPHQL_URL) + self._token = env.get("GITHUB_TOKEN") + + @property + def repo(self): + """The ``owner/name`` repository every request addresses.""" + return self._repo + + def list_open_main_prs(self): + """Every open PR based on ``main``, all pages, drafts included.""" + url = "%s/repos/%s/pulls?state=open&base=main&per_page=%d" % ( + self._api_url, self._repo, REST_PER_PAGE) + return self._fetch_json_pages(url) + + def get_pr(self, number): + """One PR record by number, whatever its state or base — the live + head source for pull_request_target events.""" + if not _is_positive_int(number): + raise ApiError("invalid PR number: %r" % (number,)) + url = "%s/repos/%s/pulls/%d" % (self._api_url, self._repo, number) + body, _ = self._fetch(url) + try: + record = json.loads(body) + except ValueError as error: + raise ApiError("malformed JSON from %s" % (url,)) from error + if not isinstance(record, dict): + raise ApiError("expected a JSON object from %s" % (url,)) + return record + + def read_statuses(self, sha): + """Every commit status recorded for ``sha``, all pages.""" + url = "%s/repos/%s/commits/%s/statuses" % (self._api_url, self._repo, sha) + return self._fetch_json_pages(url) + + def create_status(self, sha, status, token): + """POST one commit status for ``sha`` with ``token`` — the status + token only; the read token never publishes. The write targets + exactly the ``sha`` the caller validated, never any other head.""" + url = "%s/repos/%s/statuses/%s" % (self._api_url, self._repo, sha) + if not token: + raise ApiError("no STATUS_TOKEN for %s" % (url,)) + self._fetch(url, payload=status, token=token) + + def fetch_reviews(self, pr): + """Every review node of PR ``pr`` (a number or a record with a + ``number`` member), following ``pageInfo.hasNextPage``.""" + nodes = [] + cursor = None + pages = 0 + while True: + page_nodes, page_info = self._fetch_reviews_page(pr, cursor) + nodes.extend(page_nodes) + if not page_info["hasNextPage"]: + return nodes + pages += 1 + if pages >= MAX_FETCH_PAGES: + raise ApiError("reviews pagination for %r exceeds %d pages" + % (pr, MAX_FETCH_PAGES)) + cursor = page_info["endCursor"] + if not isinstance(cursor, str): + raise ApiError("reviews page for %r carries no endCursor" % (pr,)) + + def _fetch_reviews_page(self, pr, cursor): + number = pr.get("number") if isinstance(pr, dict) else pr + if not isinstance(number, int) or isinstance(number, bool) or number <= 0: + raise ApiError("invalid PR number: %r" % (number,)) + owner, sep, name = self._repo.partition("/") + if not (owner and sep and name): + raise ApiError("invalid repository: %r" % (self._repo,)) + payload = { + "query": _REVIEWS_QUERY, + "variables": {"owner": owner, "name": name, "number": number, "after": cursor}, + } + body, _ = self._fetch(self._graphql_url, payload=payload) + try: + document = json.loads(body) + except ValueError as error: + raise ApiError("malformed GraphQL response for PR %s" % (number,)) from error + if not isinstance(document, dict) or document.get("errors"): + raise ApiError("GraphQL errors for PR %s: %r" % (number, document.get("errors"))) + reviews = document + for key in ("data", "repository", "pullRequest", "reviews"): + if not isinstance(reviews, dict) or not isinstance(reviews.get(key), dict): + raise ApiError("GraphQL response for PR %s misses %s" % (number, key)) + reviews = reviews[key] + page_info = reviews.get("pageInfo") + page_nodes = reviews.get("nodes") + if (not isinstance(page_info, dict) + or not isinstance(page_info.get("hasNextPage"), bool) + or not isinstance(page_nodes, list)): + raise ApiError("GraphQL reviews for PR %s miss nodes or pageInfo" % (number,)) + return page_nodes, page_info + + def _build_request(self, url, payload, token=None): + """One request. ``token`` overrides the read token — only the + status write passes it (STATUS_TOKEN); reads stay on GITHUB_TOKEN.""" + headers = {"Accept": "application/vnd.github+json"} + auth = self._token if token is None else token + if auth: + headers["Authorization"] = "Bearer " + auth + data = None + method = "GET" + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + method = "POST" + return urllib.request.Request(url, data=data, headers=headers, method=method) + + def _fetch(self, url, payload=None, token=None): + """One request with retries. Network errors, 5xx responses and + rate-limited responses get up to MAX_ATTEMPTS attempts (a 429 + needs a Retry-After header to count as rate-limited; a 403 needs + ``x-ratelimit-remaining`` 0); every other 4xx exactly one. A + rate-limit window beyond the honor cap fails fast instead of + retrying. Returns (body_bytes, link_header).""" + failure = None + rate_limited_last = False + for attempt in range(1, MAX_ATTEMPTS + 1): + request = self._build_request(url, payload, token) + window = None + try: + response = self._transport(request) + except urllib.error.HTTPError as error: + _close_quietly(error) + if error.code < 500 and not _is_rate_limited(error.code, error.headers): + raise ApiError("HTTP %s for %s" % (error.code, url)) from error + failure = error + if _is_rate_limited(error.code, error.headers): + rate_limited_last = True + window = _rate_limit_window_seconds(error.headers) + except Exception as error: # any transport failure retries + failure = error + else: + status = response.status + headers = getattr(response, "headers", None) or {} + try: + body = response.read() + finally: + _close_quietly(response) + if 200 <= status < 300: + return body, headers.get("Link") + failure = ApiError("HTTP %s for %s" % (status, url)) + if status < 500 and not _is_rate_limited(status, headers): + break + if _is_rate_limited(status, headers): + rate_limited_last = True + window = _rate_limit_window_seconds(headers) + if attempt < MAX_ATTEMPTS: + if window is not None and window > _MAX_RETRY_AFTER_SECONDS: + raise RateLimitError( + "rate limit for %s: window %d s exceeds the %d s honor cap" + % (url, int(window), _MAX_RETRY_AFTER_SECONDS)) from failure + if window is not None: + time.sleep(min(window, _MAX_RETRY_AFTER_SECONDS)) + else: + time.sleep(_RETRY_BACKOFF_SECONDS * attempt) + if rate_limited_last: + raise RateLimitError("rate limit for %s after %d attempts" + % (url, MAX_ATTEMPTS)) from failure + if isinstance(failure, ApiError): + raise failure + raise ApiError("transport failure for %s" % (url,)) from failure + + def _fetch_json_pages(self, url): + """Every page of a REST list endpoint, following the Link header's + rel="next" URL until a page reports none. More than + MAX_FETCH_PAGES pages fails the whole read.""" + items = [] + start_url = url + pages = 0 + while url is not None: + body, link = self._fetch(url) + try: + page = json.loads(body) + except ValueError as error: + raise ApiError("malformed JSON from %s" % (url,)) from error + if not isinstance(page, list): + raise ApiError("expected a JSON array from %s" % (url,)) + items.extend(page) + url = _next_link(link, self._api_url) + pages += 1 + if url is not None and pages >= MAX_FETCH_PAGES: + raise ApiError("REST pagination from %s exceeds %d pages" + % (start_url, MAX_FETCH_PAGES)) + return items + + +def normalize_review(node): + """Map one GraphQL review node to the contract's normalized record. + + Every value is carried raw — ``lastEditedAt`` keeps its exact value, + never a boolean — and a null author maps to a null ``author_typename``. + Bot identity comes only from ``author.__typename`` and + ``author.databaseId``; REST-named fields are never consulted. A node + that is not a JSON object maps to an empty record, which the policy + validators report as invalid input rather than skipping it. + """ + if not isinstance(node, dict): + node = {} + author = node.get("author") + if not isinstance(author, dict): + author = {} + commit = node.get("commit") + return { + "id": node.get("databaseId"), + "author_typename": author.get("__typename"), + "author_id": author.get("databaseId"), + "author_login": author.get("login"), + "state": node.get("state"), + "body": node.get("body"), + "commit_oid": commit.get("oid") if isinstance(commit, dict) else None, + "submitted_at": node.get("submittedAt"), + "last_edited_at": node.get("lastEditedAt"), + } + + +def read_sha_snapshot(sha, client): + """Read snapshot S(i) for one head SHA through the validated seam. + + Returns ``{"sha", "api_error", "invalid_input", "prs", "reviews", + "outcomes"}``: ``prs`` holds the validated P(H) members (open ``main`` + PRs whose head SHA equals ``sha``), ``reviews`` maps each member's + number to its normalized records and ``outcomes`` to its policy + outcome, and the two flag lists carry every api-error/invalid-input + location (``"pulls"``, ``"pr-/reviews"``, ``"pr-record:"`, + ``"pr-"``). Failures and malformed inputs are flagged, never + skipped or truncated: a malformed PR record cannot be scoped, so it + flags the snapshot invalid-input instead of letting a shrunk P(H) + reveal a PASS. + """ + snapshot = { + "sha": sha, + "api_error": [], + "invalid_input": [], + "prs": [], + "reviews": {}, + "outcomes": {}, + } + try: + open_prs = client.list_open_main_prs() + except RateLimitError: + snapshot["api_error"].append("pulls:rate-limit") + return snapshot + except ApiError: + snapshot["api_error"].append("pulls") + return snapshot + for pr in open_prs: + if not isinstance(pr, dict): + snapshot["invalid_input"].append("pr-record:record") + continue + field = policy.validate_pr_record(pr) + if field is not None: + snapshot["invalid_input"].append("pr-record:" + field) + continue + if pr["head"]["sha"] != sha: + continue + number = pr["number"] + snapshot["prs"].append(pr) + try: + records = [normalize_review(node) for node in client.fetch_reviews(pr)] + except RateLimitError: + snapshot["api_error"].append("pr-%d/reviews:rate-limit" % (number,)) + snapshot["outcomes"][number] = "api-error" + continue + except ApiError: + snapshot["api_error"].append("pr-%d/reviews" % (number,)) + snapshot["outcomes"][number] = "api-error" + continue + snapshot["reviews"][number] = records + result = policy.outcome(records, head=sha) + snapshot["outcomes"][number] = result + if result == "invalid-input": + snapshot["invalid_input"].append("pr-%d" % (number,)) + return snapshot + + +def _decision_payload(snapshot, repo): + """The decision D(i) of one validated snapshot, as the status payload + the publisher will carry: state, reason, governing PR, description, + target URL and context. + + api-error and invalid-input flags propagate as error decisions. A + member PR's flags already carry its own outcome; the snapshot-level + locations — a failed pulls read (``pulls``) or a malformed PR record + (``pr-record:``) — name no PR, so they force the error state on + their own, even when every scoped member evaluated cleanly. A + snapshot-level error with no member names no PR at all: + ``policy.describe`` and ``policy.target_url`` require an API PR + number, none exists, so description and target URL stay None. + + Returns None when P(H) was read successfully and is empty: no PR + shares the head, so no decision exists and nothing may publish. + """ + outcomes = snapshot["outcomes"] + base = policy.decide(outcomes) if outcomes else None + level_api_flags = [flag for flag in snapshot["api_error"] if flag.startswith("pulls")] + level_invalid_flags = [flag for flag in snapshot["invalid_input"] + if str(flag).startswith("pr-record:")] + level_flags = level_api_flags + level_invalid_flags + if base is None and not level_flags: + return None + if level_flags: + # A snapshot-level flag names no PR; it forces error even when + # every scoped member evaluated cleanly. The reason names the + # class that forced the level; a failed pulls read outranks a + # malformed record when both are present. + reason = "api-error" if level_api_flags else "invalid-input" + governing_pr = base["governing_pr"] if base is not None else None + else: + reason = base["reason"] + governing_pr = base["governing_pr"] + state = "error" if level_flags else base["state"] + detail = None + if state == "error": + flags = snapshot["api_error"] if reason == "api-error" else snapshot["invalid_input"] + detail = ", ".join(sorted(str(flag) for flag in flags)) or None + description = None + review_id = None + target_url = None + if governing_pr is not None: + description = policy.describe(reason, governing_pr, detail) + # missing, api-error and invalid-input have no governing review: + # an error names no single review, missing means none exists. + if reason not in ("missing", "api-error", "invalid-input"): + latest = policy.latest_candidate( + snapshot["reviews"].get(governing_pr, []), head=snapshot["sha"]) + review_id = latest.get("id") if latest is not None else None + target_url = policy.target_url(repo, governing_pr, review_id) + return { + "state": state, + "reason": reason, + "governing_pr": governing_pr, + "description": description, + "target_url": target_url, + "context": policy.STATUS_CONTEXT, + } + + +def revalidated_decision(sha, client): + """The validated decision for one head SHA: the contract's snapshot + algorithm minus the write step. + + Reads one snapshot through ``read_sha_snapshot`` (P(H) plus the + normalized reviews and error flags) and computes the decision with + the policy. Return values only, no side effects: no status is read + or written here — ``_publish_status`` does that on the returned + decision. + + - ``("empty", None)`` — P(H) was read successfully and is empty. + - ``("stable", D)`` — D is the decision of the snapshot. + """ + snapshot = read_sha_snapshot(sha, client) + decision = _decision_payload(snapshot, client.repo) + if decision is None: + return ("empty", None) + return ("stable", decision) + + +def _selected_app_status(statuses): + """The ``bits-review/head`` status created by the gate's publisher + app with the greatest (created_at, id), or None when none exists. + + Statuses from any other creator are never selected — they are not + this gate's output and must not satisfy its comparison. A record + without a string ``created_at`` or a positive-int ``id`` cannot be + ordered, so it is never selected: a malformed entry cannot suppress + a needed write. GitHub ``created_at`` values are uniform UTC + ISO-8601 strings, where lexicographic order equals chronological + order. + """ + selected = None + selected_key = None + for status in statuses: + if not isinstance(status, dict) or status.get("context") != policy.STATUS_CONTEXT: + continue + creator = status.get("creator") + if not isinstance(creator, dict) or creator.get("login") != STATUS_CREATOR: + continue + created = status.get("created_at") + status_id = status.get("id") + if not isinstance(created, str) or not _is_positive_int(status_id): + continue + key = (created, status_id) + if selected_key is None or key > selected_key: + selected = status + selected_key = key + return selected + + +def _matches_decision(status, decision): + """True when the stored status equals the decision in every field + the write would carry: state, description, target URL and context.""" + return (status.get("state") == decision["state"] + and status.get("description") == decision["description"] + and status.get("target_url") == decision["target_url"] + and status.get("context") == decision["context"]) + + +def _status_payload(decision): + """The POST body: state and context always, the optional fields only + when the decision carries them — a member-less error names neither a + PR number nor a review, so it omits both rather than inventing + values.""" + payload = {"state": decision["state"], "context": decision["context"]} + if decision["description"] is not None: + payload["description"] = decision["description"] + if decision["target_url"] is not None: + payload["target_url"] = decision["target_url"] + return payload + + +def _publish_status(sha, client, status_token): + """Publish the validated decision for one head SHA and return the + process exit code. + + The seam's ``empty`` return exits 0 without a write. For a decision + the latest app status is + selected and compared: an exact match skips the POST (exit 0 for a + non-error decision — an ``error`` decision still exits 1 even though + its POST is suppressed), and a failed status read never suppresses + the write. A published non-error decision exits 0; a published + ``error`` decision and any failed write exit 1. Success — a POST + carrying ``state: success`` and exit 0 — can only ever come from the + all-pass conjunction branch. + """ + kind, decision = revalidated_decision(sha, client) + if kind == "empty": + return 0 + try: + statuses = client.read_statuses(sha) + except ApiError: + statuses = None # a failed read never suppresses a safety update + if statuses is not None: + current = _selected_app_status(statuses) + if current is not None and _matches_decision(current, decision): + return 1 if decision["state"] == "error" else 0 + try: + client.create_status(sha, _status_payload(decision), status_token) + except ApiError: + return 1 + return 1 if decision["state"] == "error" else 0 + + +def _is_positive_int(value): + """True for a genuine positive int; bool is an int subclass and excluded.""" + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def _env_int(env, name): + """The positive-int value of one env entry; anything else is invalid + input. Env values arrive as strings; genuine ints are accepted for + in-process callers.""" + value = env.get(name) + if _is_positive_int(value): + return value + try: + number = int(str(value)) + except (TypeError, ValueError): + number = None + if not _is_positive_int(number): + raise ResolveError("invalid %s: %r" % (name, value)) + return number + + +def _env_sha(env, name): + value = env.get(name) + if not isinstance(value, str) or _SHA_RE.fullmatch(value) is None: + raise ResolveError("invalid %s: %r" % (name, value)) + return value + + +def _env_str(env, name): + value = env.get(name) + if not isinstance(value, str) or not value: + raise ResolveError("invalid %s: %r" % (name, value)) + return value + + +def _env_pr_numbers(env, name): + """The JSON list of PR numbers; null members (GitHub's ``toJSON([*])`` + padding) are dropped, every other member must be a positive int.""" + raw = env.get(name) + try: + numbers = json.loads(raw) if isinstance(raw, str) else None + except ValueError: + numbers = None + if not isinstance(numbers, list): + raise ResolveError("invalid %s: %r" % (name, raw)) + numbers = [n for n in numbers if n is not None] + if not all(_is_positive_int(n) for n in numbers): + raise ResolveError("invalid %s: %r" % (name, raw)) + return numbers + + +def _pr_selector(pr): + """(head repository full name, head branch) of a PR record, or None + when either member is missing or not a string. Selector values are used + only in comparisons, never in commands or URLs.""" + head = pr.get("head") if isinstance(pr, dict) else None + if not isinstance(head, dict): + return None + repo = head.get("repo") + full_name = repo.get("full_name") if isinstance(repo, dict) else None + branch = head.get("ref") + if not (isinstance(full_name, str) and isinstance(branch, str)): + return None + return (full_name, branch) + + +def _require_pr_record(pr): + record = pr if isinstance(pr, dict) else {} + field = policy.validate_pr_record(record) + if field is not None: + raise ResolveError("invalid PR record: %s" % (field,)) + return record + + +def _validated_open_main_prs(client): + """The open-main PR list with every record validated. A malformed + record cannot be scoped or matched, and skipping it could leave its + head unmonitored, so it fails the whole resolve.""" + return [_require_pr_record(pr) for pr in client.list_open_main_prs()] + + +def _note_unmatched_summary(env, selector, numbers): + """Record the unmatched workflow_run selector in the job summary, + best effort: a summary-write failure must never change the exit code + or stdout.""" + path = env.get("GITHUB_STEP_SUMMARY") + if not path: + return + try: + with open(path, "a", encoding="utf-8") as handle: + handle.write("bits gate: workflow_run selectors matched no open-main PR; " + "publishing the run head only (numbers=%r)\n" % (numbers,)) + except OSError: + pass + + +def resolve_shas(client, env): + """The head SHAs to publish for this event, deduplicated and sorted. + + Every input arrives through ``env`` — the event name and its + selectors; nothing is interpolated into a command or executed. Invalid + input raises ResolveError and an API failure raises ApiError after + the client's retries; more than MAX_RESOLVE_SHAS distinct SHAs raises + ResolveError instead of truncating. + """ + event = env.get("GITHUB_EVENT_NAME") + if event == "pull_request_target": + # A title/body edit cannot change any decision input (the policy + # reads reviews, head SHA, membership/state), so it resolves to + # nothing; the guard fails open — any base-change signal or other + # action resolves normally. + action = env.get("GITHUB_PR_ACTION", "") + base_ref = env.get("GITHUB_CHANGES_BASE_REF", "") + base_sha = env.get("GITHUB_CHANGES_BASE_SHA", "") + if action == "edited" and not base_ref and not base_sha: + return [] + number = _env_int(env, "GITHUB_PR_NUMBER") + pr = client.get_pr(number) + _require_pr_record(pr) + shas = {pr["head"]["sha"]} + elif event == "workflow_run": + head_sha = _env_sha(env, "GITHUB_HEAD_SHA") + selector = (_env_str(env, "GITHUB_HEAD_REPOSITORY"), + _env_str(env, "GITHUB_HEAD_BRANCH")) + numbers = _env_pr_numbers(env, "GITHUB_PULL_REQUEST_NUMBERS") + open_prs = _validated_open_main_prs(client) + matched = [pr for pr in open_prs + if pr["number"] in numbers or _pr_selector(pr) == selector] + if matched: + selected = matched + elif numbers: + # Selectors that match nothing publish the run head alone; the + # 30-minute schedule sweep recovers any missed head. + _note_unmatched_summary(env, selector, numbers) + selected = [] + else: + # Fork runs carry no PR-number list; the full open-main + # fallback keeps every head monitored. + selected = open_prs + shas = {head_sha} | {pr["head"]["sha"] for pr in selected} + elif event in ("schedule", "workflow_dispatch"): + shas = {pr["head"]["sha"] for pr in _validated_open_main_prs(client)} + else: + raise ResolveError("unknown event name: %r" % (event,)) + if len(shas) > MAX_RESOLVE_SHAS: + raise ResolveError("%d SHAs exceed the cap of %d" % (len(shas), MAX_RESOLVE_SHAS)) + return sorted(shas) + + +def _write_resolved_shas(env, shas): + """The resolved SHAs as the ``shas`` step output through + GITHUB_OUTPUT, or on stdout when unset, for local runs. An I/O + failure surfaces as OSError to the caller, which exits 1.""" + payload = json.dumps(shas) + output = env.get("GITHUB_OUTPUT") + if output: + with open(output, "a", encoding="utf-8") as handle: + handle.write("shas=%s\n" % (payload,)) + else: + print(payload) + + +def _build_arg_parser(): + parser = argparse.ArgumentParser(prog="bits_review_gate.py", + description="Bits review merge gate") + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("resolve", help="resolve the head SHAs to publish") + publish = commands.add_parser( + "publish", help="publish the validated decision for one head") + publish.add_argument("--sha", required=True, + help="the 40-character head SHA to publish for") + return parser + + +def main(argv=None, env=None, transport=None): + """Run one CLI command and return the process exit code. + + Inputs reach Python only through ``env``. ``resolve`` exits 1 on + invalid input, on an API failure after the client's retries, when + more than MAX_RESOLVE_SHAS distinct SHAs would be emitted, or when + the resolved SHA list cannot be written; it exits 0 only after + writing the resolved SHA list. ``publish --sha H`` validates H (40 + lowercase hex characters) and exits 0 on a suppressed or published + non-error decision and on an empty P(H); every other path — an + invalid SHA, an error decision or a failed write — exits 1. + """ + env = os.environ if env is None else env + args = _build_arg_parser().parse_args(argv) + repo = env.get("GITHUB_REPOSITORY") or "" + owner, sep, name = repo.partition("/") + if not (owner and sep and name): + return 1 + client = GitHubClient(repo, transport=transport, env=env) + if args.command == "publish": + if _SHA_RE.fullmatch(args.sha) is None: + return 1 + return _publish_status(args.sha, client, env.get("STATUS_TOKEN")) + try: + shas = resolve_shas(client, env) + except (ResolveError, ApiError): + return 1 + try: + _write_resolved_shas(env, shas) + except OSError: + # The shas output is load-bearing for the publish job: a write + # failure fails the run like a resolve failure, without a + # traceback. + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/bits_review_gate_policy.py b/.github/scripts/bits_review_gate_policy.py new file mode 100644 index 0000000000..e1c302274c --- /dev/null +++ b/.github/scripts/bits_review_gate_policy.py @@ -0,0 +1,446 @@ +# Copyright 2026, Datadog, Inc. +"""Pure policy core for the Bits review merge gate. + +Standard library only, no I/O: the trusted-Bot allowlist, review-record and +PR-record validation, scope and candidacy classification, latest-candidate +selection, marker validation and the per-PR outcome precedence, the per-SHA +conjunction with its status payload (state, description, target URL) and +the full decision-input fingerprint. Records keep ``last_edited_at`` as +the raw adapter value; edited state derives from it and never replaces +it. The adapter calls these functions; they never call back. +""" + +import datetime +import hashlib +import json +import re + +#: The eight allowlisted Bits Bot databaseIds (design "Trusted result +#: recognition"). PASS is possible only from these identities. +TRUSTED_BOT_IDS = frozenset( + {214633350, 88084959, 261164178, 298424163, 266080212, 266080418, 266081015, 266788760} +) + +#: Review states the GitHub API may report for a review node. +REVIEW_STATES = ("PENDING", "COMMENTED", "APPROVED", "CHANGES_REQUESTED", "DISMISSED") + +_MARKERS = ("autotest-meta", "datadog-code-review", "datadog-autotest") +_HEX40 = re.compile(r"[0-9a-f]{40}") +_COMMENT_RE = re.compile(r"", re.DOTALL) +_VERDICT_RE = re.compile(r"\bverdict=(\S*)") +_ENV_MARKER_RE = re.compile(r"") +_REVIEWER_ID_MARKER = "" +_UTC = datetime.timezone.utc +_INSTANT_FLOOR = datetime.datetime.min.replace(tzinfo=_UTC) + +#: The dedicated commit-status context this gate publishes. +STATUS_CONTEXT = "bits-review/head" + +#: The contract's outcome-to-state mapping, verbatim; ``malformed:`` +#: outcomes map to failure through the prefix rule in ``outcome_state``. +OUTCOME_STATES = { + "pass": "success", + "fail": "failure", + "dismissed": "failure", + "untrusted-origin": "failure", + "edited-summary": "failure", + "missing": "pending", + "api-error": "error", + "invalid-input": "error", +} + +#: Conjunction severity of each status state; the greater value governs. +_STATE_SEVERITY = {"success": 0, "pending": 1, "failure": 2, "error": 3} + +#: The status description never exceeds this many characters. +DESCRIPTION_LIMIT = 140 + + +def _parse_iso8601(value): + """Return the parsed instant, or None when value is not a parseable + ISO-8601 string; the API's trailing Z spelling is accepted.""" + if not isinstance(value, str): + return None + text = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + return datetime.datetime.fromisoformat(text) + except ValueError: + return None + + +def _is_positive_int(value): + """True for a genuine positive int; bool is an int subclass and excluded.""" + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def is_marked(body): + """True when the body contains any of the three Bits marker substrings. + A non-string body has no determinable marker state and is not marked.""" + return isinstance(body, str) and any(marker in body for marker in _MARKERS) + + +def is_edited(record): + """True when the raw last_edited_at value is non-null; the record keeps + the raw value and this predicate never replaces it.""" + return record.get("last_edited_at") is not None + + +def is_candidate(record): + """A candidate is a Bot-authored review that is marked or edited. + Known-User reviews are ignored even with copied markers; unedited + unmarked Bot reviews are ignored.""" + return record.get("author_typename") == "Bot" and ( + is_marked(record.get("body")) or is_edited(record) + ) + + +def is_in_scope(record, head): + """True when the review binds to the exact head SHA: commit_oid equals + head, state is not PENDING and submitted_at is non-null. A null commit + is out of scope.""" + return ( + record.get("commit_oid") is not None + and record.get("commit_oid") == head + and record.get("state") != "PENDING" + and record.get("submitted_at") is not None + ) + + +def validate_review_record(record): + """Return None when the record passes the contract's field checks, else a + token naming the first failed field. A User-authored record is accepted + without further checks: the wiring ignores it rather than trusting it. + A marked or edited record whose author type is neither Bot nor User + cannot be classified and fails validation.""" + typename = record.get("author_typename") + if typename == "User": + return None + if not isinstance(record.get("body"), str): + return "body" + edited_raw = record.get("last_edited_at") + if edited_raw is not None and _parse_iso8601(edited_raw) is None: + return "last_edited_at" + if not _is_positive_int(record.get("id")): + return "id" + state = record.get("state") + if state not in REVIEW_STATES: + return "state" + commit_oid = record.get("commit_oid") + if commit_oid is not None and ( + not isinstance(commit_oid, str) or _HEX40.fullmatch(commit_oid) is None + ): + return "commit_oid" + submitted_at = record.get("submitted_at") + if submitted_at is None: + if state != "PENDING": + return "submitted_at" + elif _parse_iso8601(submitted_at) is None: + return "submitted_at" + if typename == "Bot" and not _is_positive_int(record.get("author_id")): + return "author_id" + if typename not in ("Bot", "User") and (is_marked(record.get("body")) or edited_raw is not None): + return "author_typename" + return None + + +def validate_pr_record(pr): + """Return None when the PR record carries the contract's required + fields — positive-int number, 40-hex head.sha, str base.ref, str + state — else a token naming the first failed field.""" + if not _is_positive_int(pr.get("number")): + return "number" + head = pr.get("head") + sha = head.get("sha") if isinstance(head, dict) else None + if not isinstance(sha, str) or _HEX40.fullmatch(sha) is None: + return "head.sha" + base = pr.get("base") + base_ref = base.get("ref") if isinstance(base, dict) else None + if not isinstance(base_ref, str): + return "base.ref" + if not isinstance(pr.get("state"), str): + return "state" + return None + + +def _utc_instant(value): + """The parsed instant normalized to UTC, or None when unparseable; a + naive spelling reads as UTC.""" + parsed = _parse_iso8601(value) + if parsed is None: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=_UTC) + return parsed.astimezone(_UTC) + + +def _possible_candidate(record): + """True when the record could be a candidate: it is marked or edited, or + its marker or edit state cannot be determined (non-string body, + unparseable edit value). Only such records can hide a governing review.""" + body = record.get("body") + if not isinstance(body, str): + return True + edited_raw = record.get("last_edited_at") + if edited_raw is not None and _parse_iso8601(edited_raw) is None: + return True + return is_marked(body) or edited_raw is not None + + +def outcome_state(outcome_token): + """The commit-status state for one outcome token; every + ``malformed:`` outcome is ``failure``. An unknown token is a + caller bug and raises.""" + if outcome_token in OUTCOME_STATES: + return OUTCOME_STATES[outcome_token] + if isinstance(outcome_token, str) and outcome_token.startswith("malformed:"): + return "failure" + raise ValueError("unknown outcome token: %r" % (outcome_token,)) + + +def decide(per_pr_outcomes): + """Conjoin the independent per-PR outcomes for one head SHA. + + ``per_pr_outcomes`` maps PR number to outcome token; each PR is judged + on its own reviews only. Severity error > failure > pending > success + picks the governing outcome, and the lowest PR number governs equal + severities. Returns ``{"state", "reason", "governing_pr"}``, or None + when no PR shares the head — an empty P(H) publishes nothing and is + never success. A non-positive-int PR number or an unknown outcome + token is a caller bug and raises.""" + if not per_pr_outcomes: + return None + best = None + best_key = None + for number, token in per_pr_outcomes.items(): + if not _is_positive_int(number): + raise ValueError("PR number must be a positive int: %r" % (number,)) + state = outcome_state(token) + # -number: the lowest PR number wins an equal-severity tie + key = (_STATE_SEVERITY[state], -number) + if best_key is None or key > best_key: + best, best_key = (state, number, token), key + return {"state": best[0], "governing_pr": best[1], "reason": best[2]} + + +def describe(reason, pr_number, detail=None): + """``Bits review on PR #[: ]``, capped at + DESCRIPTION_LIMIT characters. Numbers come from the API; a non-string + reason or detail, or a non-positive-int PR number, is a caller bug and + raises.""" + if not isinstance(reason, str): + raise ValueError("reason must be a string: %r" % (reason,)) + if detail is not None and not isinstance(detail, str): + raise ValueError("detail must be a string or None: %r" % (detail,)) + if not _is_positive_int(pr_number): + raise ValueError("PR number must be a positive int: %r" % (pr_number,)) + text = "Bits review %s on PR #%d" % (reason, pr_number) + if detail: + text += ": %s" % (detail,) + return text[:DESCRIPTION_LIMIT] + + +def target_url(repo, pr_number, review_id=None): + """The governing review's URL when ``review_id`` is given, else the PR + URL; both are built from API numbers only, never from body links. A + non-positive-int number or review id, or an invalid repository, is a + caller bug and raises.""" + if not isinstance(repo, str) or not repo: + raise ValueError("repository must be a non-empty string: %r" % (repo,)) + if not _is_positive_int(pr_number): + raise ValueError("PR number must be a positive int: %r" % (pr_number,)) + url = "https://github.com/%s/pull/%d" % (repo, pr_number) + if review_id is None: + return url + if not _is_positive_int(review_id): + raise ValueError("review id must be a positive int: %r" % (review_id,)) + return url + "#pullrequestreview-%d" % (review_id,) + + +def _jsonable(value): + """A deterministic JSON-serializable stand-in for any raw value. + Adapter values are JSON already; non-JSON values reduce to their type + name so they still change the fingerprint by type.""" + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, dict): + return {repr(key): _jsonable(item) + for key, item in sorted(value.items(), key=lambda pair: repr(pair[0]))} + return {"$type": type(value).__name__} + + +def _body_digest(body): + """sha256 of a string body; any other raw value keeps its own typed + canonical form so distinct non-string bodies still differ.""" + if isinstance(body, str): + return "sha256:" + hashlib.sha256(body.encode("utf-8")).hexdigest() + return "non-str:" + json.dumps(_jsonable(body), sort_keys=True, separators=(",", ":")) + + +def _pr_member(pr): + """The fingerprint tuple of one P(H) member: (number, head.sha, + base.ref, state), every value raw after type coercion.""" + if not isinstance(pr, dict): + pr = {} + head = pr.get("head") + base = pr.get("base") + return [ + _jsonable(pr.get("number")), + _jsonable(head.get("sha") if isinstance(head, dict) else None), + _jsonable(base.get("ref") if isinstance(base, dict) else None), + _jsonable(pr.get("state")), + ] + + +def _pr_sort_key(pr): + number = pr.get("number") if isinstance(pr, dict) else None + # Bucket 1 canonically self-orders via repr so invalid entries never + # fall back to input order (same idiom as _sorted_flags). + return (0, number) if _is_positive_int(number) else (1, repr(number)) + + +def _record_member(record): + """The fingerprint form of one normalized review record: every raw + field after type coercion, the body as a digest. The raw + ``last_edited_at`` value is kept, never a boolean.""" + if not isinstance(record, dict): + record = {} + return { + "id": _jsonable(record.get("id")), + "author_typename": _jsonable(record.get("author_typename")), + "author_id": _jsonable(record.get("author_id")), + "author_login": _jsonable(record.get("author_login")), + "state": _jsonable(record.get("state")), + "body": _body_digest(record.get("body")), + "commit_oid": _jsonable(record.get("commit_oid")), + "submitted_at": _jsonable(record.get("submitted_at")), + "last_edited_at": _jsonable(record.get("last_edited_at")), + } + + +def _record_sort_key(record): + record_id = record.get("id") if isinstance(record, dict) else None + # Bucket 1 canonically self-orders via repr so invalid entries never + # fall back to input order (same idiom as _sorted_flags). + return (0, record_id) if _is_positive_int(record_id) else (1, repr(record_id)) + + +def _sorted_flags(flags): + """Canonical flag list: every location, deterministically ordered — + flag order is fetch order, not a decision input.""" + values = [_jsonable(flag) for flag in (flags if isinstance(flags, list) else [])] + values.sort(key=repr) + return values + + +def _fingerprint_payload(snapshot): + """The canonical fingerprint structure of one SHA snapshot: the sorted + P(H) members, every member's records sorted by id, and the error flags + with their locations. Nothing partial: non-list prs or reviews values + are carried raw instead of being dropped. The head is bound by each + member's head.sha; the snapshot's own sha key is not an independent + input and never reaches the digest.""" + if not isinstance(snapshot, dict): + snapshot = {} + prs = snapshot.get("prs") + members = [_pr_member(pr) for pr in sorted(prs, key=_pr_sort_key)] if isinstance(prs, list) else _jsonable(prs) + reviews = snapshot.get("reviews") + review_members = {} + if isinstance(reviews, dict): + for number, records in reviews.items(): + if isinstance(records, list): + review_members[repr(number)] = [ + _record_member(record) for record in sorted(records, key=_record_sort_key)] + else: + review_members[repr(number)] = _jsonable(records) + return { + "prs": members, + "reviews": review_members, + "api_error": _sorted_flags(snapshot.get("api_error")), + "invalid_input": _sorted_flags(snapshot.get("invalid_input")), + } + + +def fingerprint(snapshot): + """sha256 over canonical JSON (sorted keys, no extra whitespace) of + every decision input of one SHA snapshot: the sorted P(H) members as + (number, head.sha, base.ref, state), each member's records sorted by + id with all raw fields (the body as a sha256 digest), and every + api-error/invalid-input flag with its location. Derived outcomes, the + snapshot's own sha key and unrelated PR fields never reach the digest; + a decision computed on a snapshot whose fingerprint differs from the + next one's is never reused.""" + canonical = json.dumps(_fingerprint_payload(snapshot), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _validate_markers(body): + """Marker validation in the contract's order. Returns (sub-reason, + verdict): sub-reason is None only for valid markers, and then verdict is + exactly PASS or FAIL. Delivery markers, badges and image text are never + consulted.""" + meta = [c for c in _COMMENT_RE.findall(body) if "autotest-meta" in c] + if len(meta) != 1: + return "missing-meta", None + verdicts = _VERDICT_RE.findall(meta[0]) + if len(verdicts) > 1: + return "conflicting-verdict", None + if len(verdicts) != 1 or verdicts[0] not in ("PASS", "FAIL"): + return "unknown-verdict", None + if body.count(_REVIEWER_ID_MARKER) != 1: + return "reviewer-id", None + if _ENV_MARKER_RE.findall(body) != ["prod"]: + return "env-marker", None + return None, verdicts[0] + + +def latest_candidate(records, head=None): + """The in-scope candidate with the greatest (UTC submitted_at, id). + Selection precedes any validity, dismissal, origin, edit or marker check: + the returned record may be any of those. A None head scopes out every + record, so outcome() maps to missing — un-set/pending, not fail-closed: + an error state here would block every unrelated head in the run.""" + best = None + best_key = None + for record in records: + if not (is_candidate(record) and is_in_scope(record, head)): + continue + # Unparseable or invalid ids rank lowest; rule 1 already reported + # every possible candidate carrying them, so this never changes an + # outcome. The canonical member repr breaks remaining ties so + # selection never depends on list order. + rid = record.get("id") if _is_positive_int(record.get("id")) else -1 + member = _record_member(record) + canonical = json.dumps(member, sort_keys=True, separators=(",", ":")) + key = (_utc_instant(record.get("submitted_at")) or _INSTANT_FLOOR, rid, + canonical) + if best_key is None or key > best_key: + best, best_key = record, key + return best + + +def outcome(records, validation=validate_review_record, head=None, api_error=False): + """The per-PR outcome token, the contract's rules 1-7 in exact order. + records holds every normalized review record of the PR; validation is a + callable returning validate_review_record's result for one record; head + is the PR's 40-hex head SHA and None scopes out every record; api_error + marks a failed review fetch, so no partial data is ever trusted.""" + if api_error: + return "api-error" + for record in records: + if _possible_candidate(record) and validation(record) is not None: + return "invalid-input" + latest = latest_candidate(records, head) + if latest is None: + return "missing" + if latest.get("state") == "DISMISSED": + return "dismissed" + if latest.get("author_typename") != "Bot" or latest.get("author_id") not in TRUSTED_BOT_IDS: + return "untrusted-origin" + if latest.get("last_edited_at") is not None: + return "edited-summary" + sub_reason, verdict = _validate_markers(latest.get("body")) + if sub_reason is not None: + return "malformed:" + sub_reason + return "fail" if verdict == "FAIL" else "pass" diff --git a/.github/workflows/bits-review-event.yml b/.github/workflows/bits-review-event.yml new file mode 100644 index 0000000000..5b74f6e6bd --- /dev/null +++ b/.github/workflows/bits-review-event.yml @@ -0,0 +1,17 @@ +name: Bits review event + +on: + pull_request_review: + types: [submitted, edited, dismissed] + pull_request: + types: [edited] + +permissions: {} + +jobs: + event: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Record the review event + run: echo "Bits review event received; the gate publisher re-reads live GitHub state" diff --git a/.github/workflows/bits-review-gate.yml b/.github/workflows/bits-review-gate.yml new file mode 100644 index 0000000000..2a07c97503 --- /dev/null +++ b/.github/workflows/bits-review-gate.yml @@ -0,0 +1,82 @@ +name: Bits review gate + +on: + pull_request_target: + types: [opened, reopened, synchronize, edited, closed] + workflow_run: + workflows: ["Bits review event"] + types: [completed] + schedule: + - cron: '*/30 * * * *' + workflow_dispatch: + +permissions: {} + +jobs: + resolve: + # pull_request_target sets github.ref to the base branch, so the guard + # is live for it too; the other triggers already run on refs/heads/main. + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + shas: ${{ steps.resolve.outputs.shas }} + steps: + - name: Checkout trusted resolver + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/heads/main + persist-credentials: false + + - name: Resolve the head SHAs to publish + id: resolve + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + GITHUB_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} + GITHUB_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + GITHUB_PULL_REQUEST_NUMBERS: ${{ toJSON(github.event.workflow_run.pull_requests[*].number) }} + GITHUB_PR_ACTION: ${{ github.event.action }} + GITHUB_CHANGES_BASE_REF: ${{ github.event.changes.base.ref.from }} + GITHUB_CHANGES_BASE_SHA: ${{ github.event.changes.base.sha.from }} + run: python3 .github/scripts/bits_review_gate.py resolve + + publish: + needs: resolve + if: github.ref == 'refs/heads/main' && needs.resolve.outputs.shas != '[]' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + statuses: read + id-token: write + strategy: + fail-fast: false + matrix: + sha: ${{ fromJSON(needs.resolve.outputs.shas) }} + concurrency: + group: bits-review-gate-${{ matrix.sha }} + cancel-in-progress: false + steps: + - name: Checkout trusted publisher + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/heads/main + persist-credentials: false + + - name: Mint the status token + id: octo-sts + uses: DataDog/dd-octo-sts-action@c85802a58e202b7e764b5441e4378c76043c8108 # v1.0.5 + with: + scope: DataDog/java-profiler + policy: self.bits-review-gate.publish-status + + - name: Publish the status for the resolved head + env: + SHA: ${{ matrix.sha }} + GITHUB_TOKEN: ${{ github.token }} + STATUS_TOKEN: ${{ steps.octo-sts.outputs.token }} + run: python3 .github/scripts/bits_review_gate.py publish --sha "$SHA" From e1529c361bef8cad8e657124ef9df4884b4720fa Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 13:57:34 +0100 Subject: [PATCH 02/11] Gate tests: hermetic runner, policy, publish and workflow suites --- .../scripts/tests/bits_review_gate_testlib.py | 123 ++ .../review-pr968-5872101609.json | 1 + .../review-pr985-5872953227.json | 1 + .../scripts/tests/test_bits_review_gate.sh | 25 + .../tests/test_bits_review_gate_api.py | 1178 +++++++++++++++++ .../tests/test_bits_review_gate_policy.py | 884 +++++++++++++ .../tests/test_bits_review_gate_publish.py | 698 ++++++++++ .../tests/test_bits_review_gate_workflows.py | 634 +++++++++ 8 files changed, 3544 insertions(+) create mode 100644 .github/scripts/tests/bits_review_gate_testlib.py create mode 100644 .github/scripts/tests/fixtures/bits_review_gate/review-pr968-5872101609.json create mode 100644 .github/scripts/tests/fixtures/bits_review_gate/review-pr985-5872953227.json create mode 100755 .github/scripts/tests/test_bits_review_gate.sh create mode 100644 .github/scripts/tests/test_bits_review_gate_api.py create mode 100644 .github/scripts/tests/test_bits_review_gate_policy.py create mode 100644 .github/scripts/tests/test_bits_review_gate_publish.py create mode 100644 .github/scripts/tests/test_bits_review_gate_workflows.py diff --git a/.github/scripts/tests/bits_review_gate_testlib.py b/.github/scripts/tests/bits_review_gate_testlib.py new file mode 100644 index 0000000000..2e2cc99d8b --- /dev/null +++ b/.github/scripts/tests/bits_review_gate_testlib.py @@ -0,0 +1,123 @@ +# Copyright 2026, Datadog, Inc. +"""Shared hermetic scaffold for the Bits review gate test families. + +Stdlib-only infrastructure: fixture loaders, record, PR-record and +GraphQL-node builders, the REST-fixture to GraphQL-node data mapping used +to build test inputs, and a minimal injectable fake-transport base. No gate +behavior lives here. The REST to GraphQL mapping is a local data conversion for test inputs; +it claims no equivalence between the two remote APIs. +""" + +import json +import os + +FIXTURES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "bits_review_gate") +FIXTURE_FILES = { + "pr985": "review-pr985-5872953227.json", + "pr968": "review-pr968-5872101609.json", +} + + +def load_review_fixture(pr): + """Return the parsed REST review record fixture for 'pr985' or 'pr968'.""" + with open(os.path.join(FIXTURES_DIR, FIXTURE_FILES[pr]), encoding="utf-8") as f: + return json.load(f) + + +def load_review_fixtures(): + """Return both REST review record fixtures, keyed by PR.""" + return [load_review_fixture(pr) for pr in sorted(FIXTURE_FILES)] + + +def build_review_node(**overrides): + """Return a GraphQL-shaped review node with exactly the node fields the + adapter's query requests, plus per-case keyword overrides.""" + node = { + "databaseId": 1, + "state": "COMMENTED", + "body": "review body", + "submittedAt": "2026-11-12T09:30:55Z", + "lastEditedAt": None, + "commit": {"oid": "0" * 40}, + "author": {"__typename": "Bot", "login": "bot[bot]", "databaseId": 100}, + } + node.update(overrides) + return node + + +def build_review_record(**overrides): + """Return a normalized policy record with exactly the record fields the + adapter maps; last_edited_at carries the raw value, never a boolean.""" + record = { + "id": 1, + "author_typename": "Bot", + "author_id": 100, + "author_login": "bot[bot]", + "state": "COMMENTED", + "body": "review body", + "commit_oid": "0" * 40, + "submitted_at": "2026-11-12T09:30:55Z", + "last_edited_at": None, + } + record.update(overrides) + return record + + +def rest_review_to_node(rest): + """Map a REST review fixture to a GraphQL node: body stays body, + commit_id becomes commit.oid, user.id becomes author.databaseId, + user.type becomes __typename, submitted_at becomes submittedAt, and + lastEditedAt is null — the REST payload carries no edit timestamp.""" + user = rest["user"] + return { + "databaseId": rest["id"], + "state": rest["state"], + "body": rest["body"], + "submittedAt": rest["submitted_at"], + "lastEditedAt": None, + "commit": {"oid": rest["commit_id"]}, + "author": {"__typename": user["type"], "login": user["login"], "databaseId": user["id"]}, + } + + +def build_pr_record(number=985, sha=None, base_ref="main", state="open", **overrides): + """Return a pulls-list PR record carrying the fields the gate reads + (number, head.sha, base.ref, state); keyword overrides merge on top.""" + record = { + "number": number, + "state": state, + "head": {"sha": "1" * 40 if sha is None else sha}, + "base": {"ref": base_ref}, + } + record.update(overrides) + return record + + +def fixture_node(pr, **overrides): + """A GraphQL review node derived from one supplied REST fixture through + the single REST-to-GraphQL mapping; keyword overrides merge on top.""" + node = rest_review_to_node(load_review_fixture(pr)) + node.update(overrides) + return node + + +class FakeTransport: + """Minimal injectable transport base for the API and publish families. + + Serves queued responses in order, records every call for interleaving + assertions, and raises queued exceptions. Families extend this base only + for the cases they actually need, never by adding a second base. + """ + + def __init__(self, responses=()): + self._responses = list(responses) + self.calls = [] + + def __call__(self, request): + self.calls.append(request) + if not self._responses: + raise AssertionError("unexpected transport call: %r" % (request,)) + result = self._responses.pop(0) + if isinstance(result, Exception): + raise result + return result diff --git a/.github/scripts/tests/fixtures/bits_review_gate/review-pr968-5872101609.json b/.github/scripts/tests/fixtures/bits_review_gate/review-pr968-5872101609.json new file mode 100644 index 0000000000..d76d030582 --- /dev/null +++ b/.github/scripts/tests/fixtures/bits_review_gate/review-pr968-5872101609.json @@ -0,0 +1 @@ +{"id":5872101609,"node_id":"PRR_kwDOIuaENs8AAAABXgEo6Q","user":{"login":"datadog-datadog-prod-us1[bot]","id":88084959,"node_id":"MDM6Qm90ODgwODQ5NTk=","avatar_url":"https://avatars.githubusercontent.com/u/365230?v=4","gravatar_id":"","url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D","html_url":"https://github.com/apps/datadog-datadog-prod-us1","followers_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/followers","following_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/following{/other_user}","gists_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/gists{/gist_id}","starred_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/subscriptions","organizations_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/orgs","repos_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/repos","events_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/events{/privacy}","received_events_url":"https://api.github.com/users/datadog-datadog-prod-us1%5Bbot%5D/received_events","type":"Bot","user_view_type":"public","site_admin":false},"body":"\"Bits\n\n
\nMore details\n\nThe compiled-method value cross-check is intentionally DEBUG-only because its JVMTI seam changes JVM-wide state. Release builds return null from that seam, leaving release-layout correctness outside this validation.\n\n
\nWas this helpful? React 👍 or 👎\n\n\n[Open Bits AI session](https://app.datadoghq.com/code/5881d7e2-c61c-4cc6-841f-0e2f2eac2ec2?referrer=autotest-pr-comment)\n\n🤖 Bits Code Review · Commit [`1115eb5`](https://github.com/datadog/java-profiler/commit/1115eb583f475b62544d634bb12bf750ae913e04)\n\n\n\n\n\n\n","state":"COMMENTED","html_url":"https://github.com/DataDog/java-profiler/pull/968#pullrequestreview-5872101609","pull_request_url":"https://api.github.com/repos/DataDog/java-profiler/pulls/968","author_association":"CONTRIBUTOR","is_copilot_code_review":false,"copilot_code_review_effort":null,"_links":{"html":{"href":"https://github.com/DataDog/java-profiler/pull/968#pullrequestreview-5872101609"},"pull_request":{"href":"https://api.github.com/repos/DataDog/java-profiler/pulls/968"}},"submitted_at":"2026-11-12T09:30:55Z","commit_id":"1115eb583f475b62544d634bb12bf750ae913e04"} \ No newline at end of file diff --git a/.github/scripts/tests/fixtures/bits_review_gate/review-pr985-5872953227.json b/.github/scripts/tests/fixtures/bits_review_gate/review-pr985-5872953227.json new file mode 100644 index 0000000000..79f12b9e22 --- /dev/null +++ b/.github/scripts/tests/fixtures/bits_review_gate/review-pr985-5872953227.json @@ -0,0 +1 @@ +{"id":5872953227,"node_id":"PRR_kwDOIuaENs8AAAABXg4niw","user":{"login":"datadog-official[bot]","id":214633350,"node_id":"BOT_kgDODMsLhg","avatar_url":"https://avatars.githubusercontent.com/in/1358509?v=4","gravatar_id":"","url":"https://api.github.com/users/datadog-official%5Bbot%5D","html_url":"https://github.com/apps/datadog-official","followers_url":"https://api.github.com/users/datadog-official%5Bbot%5D/followers","following_url":"https://api.github.com/users/datadog-official%5Bbot%5D/following{/other_user}","gists_url":"https://api.github.com/users/datadog-official%5Bbot%5D/gists{/gist_id}","starred_url":"https://api.github.com/users/datadog-official%5Bbot%5D/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datadog-official%5Bbot%5D/subscriptions","organizations_url":"https://api.github.com/users/datadog-official%5Bbot%5D/orgs","repos_url":"https://api.github.com/users/datadog-official%5Bbot%5D/repos","events_url":"https://api.github.com/users/datadog-official%5Bbot%5D/events{/privacy}","received_events_url":"https://api.github.com/users/datadog-official%5Bbot%5D/received_events","type":"Bot","user_view_type":"public","site_admin":false},"body":"\"Bits\n\nThe validator rejects valid quarantine entries for existing tests declared in included .inc files. It also misclassifies Java class wildcards in one-segment packages and rejects undecorated GTEST_TEST IDs.\n\n[Open Bits AI session](https://app.datadoghq.com/code/3f31b9a0-15de-498f-824c-12736cc7f5c7?referrer=autotest-pr-comment)\n\n🤖 Bits Code Review · Commit [`1dcc3fa`](https://github.com/datadog/java-profiler/commit/1dcc3fa57582f2b595403f2b6a54e4464f352223)\n\n\n\n\n\n\n","state":"COMMENTED","html_url":"https://github.com/DataDog/java-profiler/pull/985#pullrequestreview-5872953227","pull_request_url":"https://api.github.com/repos/DataDog/java-profiler/pulls/985","author_association":"NONE","is_copilot_code_review":false,"copilot_code_review_effort":null,"_links":{"html":{"href":"https://github.com/DataDog/java-profiler/pull/985#pullrequestreview-5872953227"},"pull_request":{"href":"https://api.github.com/repos/DataDog/java-profiler/pulls/985"}},"submitted_at":"2026-11-12T10:38:09Z","commit_id":"1dcc3fa57582f2b595403f2b6a54e4464f352223"} \ No newline at end of file diff --git a/.github/scripts/tests/test_bits_review_gate.sh b/.github/scripts/tests/test_bits_review_gate.sh new file mode 100755 index 0000000000..94d50852a3 --- /dev/null +++ b/.github/scripts/tests/test_bits_review_gate.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2026, Datadog, Inc. + +# Hermetic runner for the Bits review gate suite: byte-compiles both gate +# scripts, then runs only this gate's Python test family. +# Run with: bash .github/scripts/tests/test_bits_review_gate.sh + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Hermetic: the suite never needs credentials, so unset both tokens; a stray +# value in the environment cannot change any test outcome. +unset GITHUB_TOKEN STATUS_TOKEN + +# Byte-compile first: a missing or syntactically broken gate script fails +# before any test runs. +python3 -m py_compile "$SCRIPT_DIR/../bits_review_gate.py" \ + "$SCRIPT_DIR/../bits_review_gate_policy.py" + +# The discover pattern runs only this gate's family; the shared helper +# bits_review_gate_testlib.py is its one allowed non-family import. +cd "$SCRIPT_DIR/../../.." +exec python3 -m unittest discover -s .github/scripts/tests -p 'test_bits_review_gate_*.py' diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py new file mode 100644 index 0000000000..b5eaa65527 --- /dev/null +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -0,0 +1,1178 @@ +# Copyright 2026, Datadog, Inc. +"""API-layer tests for the Bits review gate GitHubClient. + +Pagination, retries, fail-closed API-error handling and read-token +separation, plus the shared fake transport's call recording, the +normalization/validation wiring of the snapshot seam — all hermetic +through the fake transport, with no network and no credentials. +""" + +import contextlib +import io +import json +import os +import sys +import tempfile +import time +import unittest +import unittest.mock +import urllib.error +import urllib.request + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import bits_review_gate as gate # noqa: E402 +import bits_review_gate_policy as policy # noqa: E402 + +import bits_review_gate_testlib as testlib # noqa: E402 + +REPO = "DataDog/java-profiler" +API = "https://api.example.test" +GRAPHQL = "https://api.example.test/graphql" +GITHUB_TOKEN = "gh-read-token" +STATUS_TOKEN = "status-write-token" +SHA = "1" * 40 +PR985_HEAD = "1dcc3fa57582f2b595403f2b6a54e4464f352223" +PR968_HEAD = "1115eb583f475b62544d634bb12bf750ae913e04" +PULLS_URL = "%s/repos/%s/pulls?state=open&base=main&per_page=100" % (API, REPO) +STATUSES_URL = "%s/repos/%s/commits/%s/statuses" % (API, REPO, SHA) + + +PR_URL = "%s/repos/%s/pulls/985" % (API, REPO) + + +class FakeResponse: + """A recorded HTTP response: status, bytes body, optional Link header.""" + + def __init__(self, status, body=b"[]", link=None): + self.status = status + self._body = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") + self.headers = {"Link": link} if link else {} + + def read(self): + return self._body + + +class FakeStatusResponse(FakeResponse): + """A non-2xx response delivered as a normal return, carrying headers.""" + + def __init__(self, status, headers): + super().__init__(status, []) + self.headers = headers + + +class ClosingResponse(FakeResponse): + """A fake response recording whether its close() ran.""" + + def __init__(self, status, body=b"[]"): + super().__init__(status, body) + self.closed = False + + def close(self): + self.closed = True + + +class RecordingHTTPError(urllib.error.HTTPError): + """An HTTPError recording whether its close() ran.""" + + def __init__(self, url, code, msg, hdrs, fp): + super().__init__(url, code, msg, hdrs, fp) + self.closed = False + + def close(self): + self.closed = True + super().close() + + +def make_client(transport): + return gate.GitHubClient(REPO, transport=transport, env={ + "GITHUB_API_URL": API, "GITHUB_GRAPHQL_URL": GRAPHQL, + "GITHUB_TOKEN": GITHUB_TOKEN, "STATUS_TOKEN": STATUS_TOKEN}) + + +def reviews_page(nodes, has_next=False, cursor="cursor-1"): + return {"data": {"repository": {"pullRequest": {"reviews": { + "nodes": nodes, + "pageInfo": {"hasNextPage": has_next, "endCursor": cursor}}}}}} + + +class FakeTransportRecordingTest(unittest.TestCase): + """The shared fake transport records calls and surfaces queued failures.""" + + def test_records_calls_and_rejects_unexpected_ones(self): + transport = testlib.FakeTransport([OSError("socket reset")]) + request = urllib.request.Request(PULLS_URL) + with self.assertRaises(OSError): + transport(request) + with self.assertRaises(AssertionError): + transport(request) + self.assertEqual(len(transport.calls), 2) + + def test_call_sequence_is_recorded_in_order(self): + transport = testlib.FakeTransport([ + FakeResponse(200, reviews_page([], has_next=True)), + FakeResponse(200, reviews_page([])), + ]) + nodes = make_client(transport).fetch_reviews(985) + self.assertEqual(nodes, []) + after = [json.loads(call.data)["variables"]["after"] for call in transport.calls] + self.assertEqual(after, [None, "cursor-1"]) + + +class RestPaginationTest(unittest.TestCase): + """REST list endpoints are consumed page by page, following Link.""" + + def test_multi_page_pr_list_is_fully_consumed(self): + page2_url = PULLS_URL + "&page=2" + page1 = [{"number": 985}, {"number": 968, "draft": True}] + link = '<%s>; rel="next", <%s&page=3>; rel="last"' % (page2_url, PULLS_URL) + transport = testlib.FakeTransport([ + FakeResponse(200, page1, link=link), + FakeResponse(200, [{"number": 7}]), + ]) + prs = make_client(transport).list_open_main_prs() + self.assertEqual(prs, [{"number": 985}, {"number": 968, "draft": True}, {"number": 7}]) + self.assertEqual([call.full_url for call in transport.calls], [PULLS_URL, page2_url]) + + def test_page_without_next_link_ends_pagination(self): + transport = testlib.FakeTransport([FakeResponse(200, [{"number": 1}])]) + self.assertEqual(make_client(transport).list_open_main_prs(), [{"number": 1}]) + self.assertEqual(len(transport.calls), 1) + + def test_statuses_are_paginated_fully(self): + page2_url = STATUSES_URL + "?page=2" + transport = testlib.FakeTransport([ + FakeResponse(200, [{"id": 1}], link='<%s>; rel="next"' % page2_url), + FakeResponse(200, [{"id": 2}]), + ]) + self.assertEqual(make_client(transport).read_statuses(SHA), [{"id": 1}, {"id": 2}]) + self.assertEqual([call.full_url for call in transport.calls], [STATUSES_URL, page2_url]) + + def test_cross_origin_next_url_fails_closed_without_a_request(self): + transport = testlib.FakeTransport([ + FakeResponse(200, [{"number": 1}], + link='; rel="next"')]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_relative_next_url_fails_closed(self): + transport = testlib.FakeTransport([ + FakeResponse(200, [{"number": 1}], link='; rel="next"')]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_degenerate_empty_next_url_resolves_to_no_next_link(self): + self.assertIsNone(gate._next_link('<>; rel="next"', API)) + + def test_degenerate_next_url_page_yields_items_with_one_call(self): + transport = testlib.FakeTransport( + [FakeResponse(200, [{"number": 1}], link='<>; rel="next"')]) + self.assertEqual(make_client(transport).list_open_main_prs(), [{"number": 1}]) + self.assertEqual(len(transport.calls), 1) + + def test_degenerate_next_part_is_skipped_for_a_later_valid_next(self): + page2_url = PULLS_URL + "&page=2" + transport = testlib.FakeTransport([ + FakeResponse(200, [{"number": 1}], + link='<>; rel="next", <%s>; rel="next"' % page2_url), + FakeResponse(200, [{"number": 2}])]) + self.assertEqual(make_client(transport).list_open_main_prs(), + [{"number": 1}, {"number": 2}]) + self.assertEqual([call.full_url for call in transport.calls], [PULLS_URL, page2_url]) + + def test_comma_inside_a_next_url_does_not_break_parsing(self): + page2_url = PULLS_URL + "&page=2,cursor=a,b" + transport = testlib.FakeTransport([ + FakeResponse(200, [{"number": 1}], link='<%s>; rel="next"' % page2_url), + FakeResponse(200, [{"number": 2}])]) + self.assertEqual(make_client(transport).list_open_main_prs(), + [{"number": 1}, {"number": 2}]) + self.assertEqual([call.full_url for call in transport.calls], [PULLS_URL, page2_url]) + + +class GraphqlPaginationTest(unittest.TestCase): + """Review pages are followed until pageInfo.hasNextPage is false.""" + + def test_multi_page_reviews_are_fully_consumed(self): + first = testlib.build_review_node(databaseId=1) + second = testlib.build_review_node(databaseId=2) + transport = testlib.FakeTransport([ + FakeResponse(200, reviews_page([first], has_next=True, cursor="cursor-1")), + FakeResponse(200, reviews_page([second])), + ]) + self.assertEqual(make_client(transport).fetch_reviews(985), [first, second]) + after = [json.loads(call.data)["variables"]["after"] for call in transport.calls] + self.assertEqual(after, [None, "cursor-1"]) + + def test_single_page_stops_when_has_next_page_is_false(self): + node = testlib.build_review_node() + transport = testlib.FakeTransport([FakeResponse(200, reviews_page([node]))]) + self.assertEqual(make_client(transport).fetch_reviews({"number": 985}), [node]) + self.assertEqual(json.loads(transport.calls[0].data)["variables"]["number"], 985) + self.assertEqual(len(transport.calls), 1) + + def test_query_requests_exactly_the_contract_fields(self): + transport = testlib.FakeTransport([FakeResponse(200, reviews_page([]))]) + make_client(transport).fetch_reviews(985) + query = json.loads(transport.calls[0].data)["query"] + for field in ("databaseId", "state", "body", "submittedAt", "lastEditedAt", + "commit { oid }", "__typename", "login", + "on Bot { databaseId }", "on User { databaseId }"): + self.assertIn(field, query) + self.assertNotIn("userContentEdits", query) + self.assertNotIn("includesCreatedEdit", query) + + def test_invalid_pr_number_fails_without_a_request(self): + for bad in ("985", 0, True): + with self.subTest(bad=bad): + transport = testlib.FakeTransport([]) + with self.assertRaises(gate.ApiError): + make_client(transport).fetch_reviews(bad) + self.assertEqual(transport.calls, []) + + +class RetryPolicyTest(unittest.TestCase): + """Network errors, 5xx and rate-limited 403/429 retry up to three + attempts; every other 4xx never.""" + + def test_5xx_is_retried_exactly_three_times_then_fails(self): + transport = testlib.FakeTransport([FakeResponse(500)] * 3) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 3) + + def test_5xx_retry_can_succeed_on_a_later_attempt(self): + transport = testlib.FakeTransport([FakeResponse(503), FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(len(transport.calls), 2) + + def test_4xx_response_is_not_retried(self): + transport = testlib.FakeTransport([FakeResponse(404)]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_network_error_is_retried_until_success(self): + transport = testlib.FakeTransport([OSError("connection reset"), FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(len(transport.calls), 2) + + def test_network_error_exhausts_attempts_then_fails(self): + transport = testlib.FakeTransport([OSError("connection reset")] * 3) + with self.assertRaises(gate.ApiError): + make_client(transport).read_statuses(SHA) + self.assertEqual(len(transport.calls), 3) + + def test_http_errors_follow_the_same_retry_policy(self): + cases = [ + (urllib.error.HTTPError(PULLS_URL, 403, "Forbidden", None, None), 1), + (urllib.error.HTTPError(PULLS_URL, 502, "Bad Gateway", None, None), 3), + ] + for error, attempts in cases: + with self.subTest(code=error.code, attempts=attempts): + transport = testlib.FakeTransport([error] * attempts) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), attempts) + recovered = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 502, "Bad Gateway", None, None), + FakeResponse(200, [])]) + self.assertEqual(make_client(recovered).list_open_main_prs(), []) + self.assertEqual(len(recovered.calls), 2) + + def test_429_http_error_with_retry_after_is_retried_then_succeeds(self): + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", + {"Retry-After": "0"}, None), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(len(transport.calls), 2) + + def test_429_non_2xx_response_with_retry_after_is_retried_then_succeeds(self): + transport = testlib.FakeTransport([ + FakeStatusResponse(429, {"Retry-After": "0"}), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(len(transport.calls), 2) + + def test_rate_limited_403_exhausts_the_shared_budget(self): + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 403, "Forbidden", + {"x-ratelimit-remaining": "0"}, None)] * 3) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 3) + + def test_429_without_a_header_signal_fails_fast(self): + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", None, None)]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_rate_limit_headers_match_case_insensitively(self): + retried = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", + {"RETRY-AFTER": "0"}, None), + FakeResponse(200, [])]) + self.assertEqual(make_client(retried).list_open_main_prs(), []) + self.assertEqual(len(retried.calls), 2) + exhausted = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 403, "Forbidden", + {"X-RateLimit-Remaining": "0"}, None)] * 3) + with self.assertRaises(gate.ApiError): + make_client(exhausted).list_open_main_prs() + self.assertEqual(len(exhausted.calls), 3) + + def test_retry_after_is_clamped_to_the_cap(self): + # With the 300 s honor cap, a window beyond it fails fast instead + # of sleeping: exactly one attempt, RateLimitError, no long sleep. + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", + {"Retry-After": "3600"}, None)]) + with self.assertRaises(gate.RateLimitError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + self.assertEqual(slept.call_args_list, []) + + def test_negative_retry_after_is_clamped_to_zero(self): + # A hostile Retry-After must reach time.sleep only after the + # clamp: time.sleep(-1) would raise ValueError outside both + # fail-closed handlers. + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", + {"Retry-After": "-1"}, None), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(slept.call_args_list, [unittest.mock.call(0)]) + + def test_plain_403_with_retry_after_is_not_retried(self): + # A 403 without x-ratelimit-remaining 0 is never classified as a + # rate limit, whatever Retry-After header it carries. + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 403, "Forbidden", + {"Retry-After": "5"}, None)]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_403_with_retry_after_and_exhausted_remaining_is_retried(self): + # The genuine-rate-limit path: a 403 with both signals honors the + # delay and retries. + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 403, "Forbidden", + {"Retry-After": "0", + "x-ratelimit-remaining": "0"}, None), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(slept.call_args_list, [unittest.mock.call(0)]) + + def test_429_with_remaining_zero_but_no_retry_after_fails_fast(self): + # A 429 needs a Retry-After header to count as rate-limited; + # remaining 0 alone must not make it retry. + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", + {"x-ratelimit-remaining": "0"}, None)]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_retry_after_600_exceeds_the_honor_cap_and_fails_fast(self): + transport = testlib.FakeTransport([ + FakeStatusResponse(429, {"Retry-After": "600"})]) + with self.assertRaises(gate.RateLimitError) as caught: + make_client(transport).list_open_main_prs() + self.assertIn("rate limit", str(caught.exception)) + self.assertEqual(len(transport.calls), 1) + + def test_rate_limit_reset_beyond_the_cap_fails_fast(self): + transport = testlib.FakeTransport([ + FakeStatusResponse(403, {"x-ratelimit-remaining": "0", + "x-ratelimit-reset": str(int(time.time()) + 600)})]) + with self.assertRaises(gate.RateLimitError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_rate_limit_reset_within_the_cap_is_honored(self): + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + FakeStatusResponse(403, {"x-ratelimit-remaining": "0", + "x-ratelimit-reset": str(int(time.time()) + 10)}), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(len(transport.calls), 2) + slept_seconds = slept.call_args_list[0][0][0] + self.assertGreaterEqual(slept_seconds, 9.0) + self.assertLessEqual(slept_seconds, 10.0) + + def test_rate_limit_reset_in_the_past_clamps_to_zero(self): + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + FakeStatusResponse(403, {"x-ratelimit-remaining": "0", + "x-ratelimit-reset": str(int(time.time()) - 30)}), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(slept.call_args_list, [unittest.mock.call(0)]) + + def test_in_cap_retry_after_is_honored(self): + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + FakeStatusResponse(429, {"Retry-After": "120"}), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(slept.call_args_list, [unittest.mock.call(120)]) + + def test_rate_limit_exhaustion_raises_a_named_rate_limit_error(self): + transport = testlib.FakeTransport([ + FakeStatusResponse(403, {"x-ratelimit-remaining": "0"})] * 3) + with self.assertRaises(gate.RateLimitError) as caught: + make_client(transport).list_open_main_prs() + self.assertIn("rate limit for %s after %d attempts" % (PULLS_URL, gate.MAX_ATTEMPTS), + str(caught.exception)) + self.assertEqual(len(transport.calls), 3) + + def test_non_integer_retry_after_falls_back_to_default_backoff(self): + with unittest.mock.patch.object(gate.time, "sleep") as slept: + transport = testlib.FakeTransport([ + urllib.error.HTTPError(PULLS_URL, 429, "Too Many Requests", + {"Retry-After": "soon"}, None), + FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(slept.call_args_list, + [unittest.mock.call(gate._RETRY_BACKOFF_SECONDS)]) + + +class ResponseClosingTest(unittest.TestCase): + """Every response opened in _fetch is closed on every exit path.""" + + def test_success_path_closes_the_response(self): + response = ClosingResponse(200, []) + transport = testlib.FakeTransport([response]) + make_client(transport).list_open_main_prs() + self.assertTrue(response.closed) + + def test_non_2xx_path_closes_the_response(self): + response = ClosingResponse(404) + transport = testlib.FakeTransport([response]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertTrue(response.closed) + + def test_http_error_path_closes_the_error(self): + error = RecordingHTTPError(PULLS_URL, 404, "Not Found", None, None) + transport = testlib.FakeTransport([error]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertTrue(error.closed) + + def test_retried_http_error_closes_the_error(self): + error = RecordingHTTPError(PULLS_URL, 502, "Bad Gateway", None, None) + transport = testlib.FakeTransport([error, FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertTrue(error.closed) + + +class MatchesDecisionTest(unittest.TestCase): + """The suppression predicate requires every write-payload field to + match, the persisted context included.""" + + def _decision(self): + return {"state": "success", "reason": "pass", "governing_pr": 968, + "description": "Bits review: pass", + "target_url": "https://example.test/pr/968", + "context": policy.STATUS_CONTEXT} + + def _status(self, **overrides): + status = {"state": "success", "description": "Bits review: pass", + "target_url": "https://example.test/pr/968", + "context": policy.STATUS_CONTEXT} + status.update(overrides) + return status + + def test_equal_status_matches(self): + self.assertTrue(gate._matches_decision(self._status(), self._decision())) + + def test_foreign_context_never_matches(self): + self.assertFalse(gate._matches_decision( + self._status(context="bits-review/head-evil"), self._decision())) + + +class GraphqlFailClosedTest(unittest.TestCase): + """GraphQL errors and missing structure never yield a partial list.""" + + def test_errors_with_partial_data_fail(self): + page = reviews_page([testlib.build_review_node()]) + page["errors"] = [{"message": "Something went wrong"}] + transport = testlib.FakeTransport([FakeResponse(200, page)]) + with self.assertRaises(gate.ApiError): + make_client(transport).fetch_reviews(985) + self.assertEqual(len(transport.calls), 1) + + def test_missing_response_structure_fails(self): + def drop_reviews(page): + page["data"]["repository"]["pullRequest"].pop("reviews") + breakers = [ + lambda p: p.pop("data"), + lambda p: p["data"].pop("repository"), + lambda p: p["data"]["repository"].pop("pullRequest"), + drop_reviews, + lambda p: p["data"]["repository"]["pullRequest"]["reviews"].pop("nodes"), + lambda p: p["data"]["repository"]["pullRequest"]["reviews"].pop("pageInfo"), + ] + for breaker in breakers: + with self.subTest(breaker=breaker): + page = reviews_page([testlib.build_review_node()]) + breaker(page) + transport = testlib.FakeTransport([FakeResponse(200, page)]) + with self.assertRaises(gate.ApiError): + make_client(transport).fetch_reviews(985) + + def test_missing_end_cursor_on_nonfinal_page_fails(self): + page = reviews_page([testlib.build_review_node()], has_next=True, cursor=None) + transport = testlib.FakeTransport([FakeResponse(200, page)]) + with self.assertRaises(gate.ApiError): + make_client(transport).fetch_reviews(985) + + def test_malformed_json_fails_closed(self): + readers = [lambda c: c.list_open_main_prs(), + lambda c: c.read_statuses(SHA), + lambda c: c.fetch_reviews(985)] + for reader in readers: + with self.subTest(reader=reader): + transport = testlib.FakeTransport([FakeResponse(200, b"not json")]) + with self.assertRaises(gate.ApiError): + reader(make_client(transport)) + + def test_rest_non_array_body_fails(self): + transport = testlib.FakeTransport([FakeResponse(200, {"message": "nope"})]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + + +class ReadTokenTest(unittest.TestCase): + """Reads carry GITHUB_TOKEN only; STATUS_TOKEN never appears.""" + + def _recorded_reads(self): + transport = testlib.FakeTransport([ + FakeResponse(200, []), + FakeResponse(200, reviews_page([])), + FakeResponse(200, []), + ]) + client = make_client(transport) + client.list_open_main_prs() + client.fetch_reviews(985) + client.read_statuses(SHA) + return transport + + def test_read_urls_use_the_env_base_urls(self): + transport = self._recorded_reads() + self.assertEqual([call.full_url for call in transport.calls], + [PULLS_URL, GRAPHQL, STATUSES_URL]) + + def test_every_read_carries_github_token_and_never_status_token(self): + transport = self._recorded_reads() + self.assertEqual(len(transport.calls), 3) + for call in transport.calls: + self.assertEqual(call.headers.get("Authorization"), "Bearer " + GITHUB_TOKEN) + for value in call.headers.values(): + self.assertNotIn(STATUS_TOKEN, value) + + def test_missing_github_token_sends_no_authorization(self): + transport = testlib.FakeTransport([FakeResponse(200, [])]) + client = gate.GitHubClient(REPO, transport=transport, env={ + "GITHUB_API_URL": API, "GITHUB_GRAPHQL_URL": GRAPHQL, "STATUS_TOKEN": STATUS_TOKEN}) + client.list_open_main_prs() + self.assertNotIn("Authorization", transport.calls[0].headers) + + def test_default_base_urls_without_env_entries(self): + transport = testlib.FakeTransport([FakeResponse(200, []), FakeResponse(200, reviews_page([]))]) + client = gate.GitHubClient(REPO, transport=transport, env={}) + client.list_open_main_prs() + client.fetch_reviews(985) + self.assertTrue(transport.calls[0].full_url.startswith("https://api.github.com/repos/")) + self.assertEqual(transport.calls[1].full_url, "https://api.github.com/graphql") + + +class HelperBuilderTest(unittest.TestCase): + """The extended scaffold builders emit exactly their fields.""" + + def test_pr_record_builder_emits_exactly_the_gate_fields(self): + pr = testlib.build_pr_record(number=968, sha=PR968_HEAD) + self.assertEqual(pr, {"number": 968, "state": "open", + "head": {"sha": PR968_HEAD}, "base": {"ref": "main"}}) + self.assertEqual(testlib.build_pr_record()["head"]["sha"], "1" * 40) + + def test_fixture_node_wraps_the_single_mapping_with_overrides(self): + rest = testlib.load_review_fixture("pr968") + node = testlib.fixture_node("pr968", lastEditedAt="2026-06-30T08:48:47Z") + self.assertEqual(node["body"], rest["body"]) + self.assertEqual(node["commit"]["oid"], rest["commit_id"]) + self.assertEqual(node["lastEditedAt"], "2026-06-30T08:48:47Z") + + +class NormalizeReviewTest(unittest.TestCase): + """normalize_review maps GraphQL nodes to the contract's records. + + Every author value comes from the GraphQL author fields and is chosen + distinct from any REST fixture value, so a mapping that consulted a + REST-named field cannot pass. No REST/GraphQL equivalence is claimed.""" + + def test_distinct_author_types_map_to_distinct_records(self): + authors = [ + {"__typename": "Bot", "login": "graphql-bot[bot]", "databaseId": 555001}, + {"__typename": "User", "login": "graphql-user", "databaseId": 555002}, + {"__typename": "Organization", "login": "graphql-org", "databaseId": 555003}, + None, + ] + records = [ + gate.normalize_review(testlib.build_review_node(author=author, databaseId=61 + i)) + for i, author in enumerate(authors) + ] + self.assertEqual([r["author_typename"] for r in records], + ["Bot", "User", "Organization", None]) + self.assertEqual([r["author_id"] for r in records], [555001, 555002, 555003, None]) + self.assertEqual([r["author_login"] for r in records], + ["graphql-bot[bot]", "graphql-user", "graphql-org", None]) + + def test_rest_named_fields_are_never_consulted(self): + node = testlib.build_review_node( + databaseId=555004, + submittedAt="2026-05-05T05:05:05Z", + lastEditedAt="2026-05-05T06:06:06Z", + commit={"oid": PR968_HEAD}, + author={"__typename": "Bot", "login": "graphql-only-bot[bot]", "databaseId": 555005}, + user={"id": 1, "type": "User", "login": "rest-user"}, + commit_id="rest-commit-id", + submitted_at="rest-submitted-at", + last_edited_at="rest-edited-at", + ) + record = gate.normalize_review(node) + self.assertEqual(record["id"], 555004) + self.assertEqual(record["author_typename"], "Bot") + self.assertEqual(record["author_id"], 555005) + self.assertEqual(record["author_login"], "graphql-only-bot[bot]") + self.assertEqual(record["commit_oid"], PR968_HEAD) + self.assertEqual(record["submitted_at"], "2026-05-05T05:05:05Z") + self.assertEqual(record["last_edited_at"], "2026-05-05T06:06:06Z") + + def test_fixture_nodes_round_trip_to_the_exact_contract_record(self): + for pr in ("pr985", "pr968"): + with self.subTest(pr=pr): + rest = testlib.load_review_fixture(pr) + record = gate.normalize_review(testlib.rest_review_to_node(rest)) + self.assertEqual(record, testlib.build_review_record( + id=rest["id"], author_typename=rest["user"]["type"], + author_id=rest["user"]["id"], author_login=rest["user"]["login"], + state=rest["state"], body=rest["body"], + commit_oid=rest["commit_id"], submitted_at=rest["submitted_at"], + last_edited_at=None)) + + def test_raw_last_edited_at_is_retained_verbatim(self): + for raw in ("2026-06-30T08:48:47Z", "not-a-timestamp", ""): + with self.subTest(raw=raw): + record = gate.normalize_review(testlib.build_review_node(lastEditedAt=raw)) + self.assertEqual(record["last_edited_at"], raw) + self.assertIsNot(record["last_edited_at"], True) + + def test_null_and_non_object_authors_map_to_null_author_fields(self): + for author in (None, {}, "not-an-object"): + with self.subTest(author=author): + record = gate.normalize_review(testlib.build_review_node(author=author)) + self.assertIsNone(record["author_typename"]) + self.assertIsNone(record["author_id"]) + self.assertIsNone(record["author_login"]) + + def test_non_object_node_maps_to_a_fail_closed_empty_record(self): + record = gate.normalize_review("not-a-node") + self.assertEqual(record, testlib.build_review_record( + id=None, author_typename=None, author_id=None, author_login=None, + state=None, body=None, commit_oid=None, submitted_at=None, + last_edited_at=None)) + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "invalid-input") + + +class PaginationCapTest(unittest.TestCase): + """Both pagination loops fail closed past MAX_FETCH_PAGES — the read + is never truncated and never hangs. Pre-fix failure mode: without the + cap, the cap-th page's 'unexpected transport call' AssertionError is + retried by _fetch, so the message and call-count assertions fail.""" + + def test_reviews_pagination_past_the_cap_fails_closed(self): + pages = [FakeResponse(200, reviews_page([], has_next=True, cursor="c%d" % i)) + for i in range(gate.MAX_FETCH_PAGES)] + transport = testlib.FakeTransport(pages) + with self.assertRaises(gate.ApiError) as caught: + make_client(transport).fetch_reviews(985) + self.assertIn("reviews pagination", str(caught.exception)) + self.assertEqual(len(transport.calls), gate.MAX_FETCH_PAGES) + + def test_rest_pagination_past_the_cap_fails_closed(self): + def page(i): + url = PULLS_URL + "&page=%d" % (i + 2,) + return FakeResponse(200, [{"number": i}], link='<%s>; rel="next"' % url) + transport = testlib.FakeTransport([page(i) for i in range(gate.MAX_FETCH_PAGES)]) + with self.assertRaises(gate.ApiError) as caught: + make_client(transport).list_open_main_prs() + self.assertIn("REST pagination", str(caught.exception)) + self.assertEqual(len(transport.calls), gate.MAX_FETCH_PAGES) + + def test_read_ending_on_the_cap_th_page_succeeds(self): + pages = [FakeResponse(200, reviews_page([], has_next=True, cursor="c%d" % i)) + for i in range(gate.MAX_FETCH_PAGES - 1)] + pages.append(FakeResponse(200, reviews_page([]))) + transport = testlib.FakeTransport(pages) + self.assertEqual(make_client(transport).fetch_reviews(985), []) + self.assertEqual(len(transport.calls), gate.MAX_FETCH_PAGES) + +class SnapshotReadTest(unittest.TestCase): + """The fetch path wires the T1 validators in, fail-closed: invalid + possible candidates and malformed PR records are flagged, never + skipped, so an older PASS is never uncovered.""" + + def _read(self, open_prs, review_pages, head=PR968_HEAD): + transport = testlib.FakeTransport( + [FakeResponse(200, open_prs)] + list(review_pages)) + return gate.read_sha_snapshot(head, make_client(transport)), transport + + def test_fixture_reviews_round_trip_and_classify_through_the_fetch_path(self): + for pr, number, head, expected in ( + ("pr968", 968, PR968_HEAD, "pass"), ("pr985", 985, PR985_HEAD, "fail")): + with self.subTest(pr=pr): + rest = testlib.load_review_fixture(pr) + snapshot, _ = self._read( + [testlib.build_pr_record(number=number, sha=head)], + [FakeResponse(200, reviews_page([testlib.rest_review_to_node(rest)]))], + head=head) + self.assertEqual(snapshot["outcomes"], {number: expected}) + self.assertEqual(snapshot["invalid_input"], []) + self.assertEqual(snapshot["api_error"], []) + record = snapshot["reviews"][number][0] + self.assertEqual(record["body"], rest["body"]) + self.assertEqual(record["commit_oid"], rest["commit_id"]) + self.assertEqual(record["submitted_at"], rest["submitted_at"]) + self.assertIsNone(record["last_edited_at"]) + + def test_each_invalid_input_shape_flags_the_pr_outcome_over_an_older_pass(self): + older_pass = testlib.fixture_node("pr968") + marked_body = testlib.load_review_fixture("pr985")["body"] + shapes = { + "state": {"state": "CHANGED_REQUESTED"}, + "submitted-at": {"submittedAt": "garbage"}, + "null-submitted-at": {"submittedAt": None}, + "last-edited-at": {"lastEditedAt": "not-a-timestamp", "body": "plain bot text"}, + "id": {"databaseId": 0}, + "string-id": {"databaseId": "5873000001"}, + "bot-database-id": {"author": {"__typename": "Bot", "login": "graphql-bot[bot]", "databaseId": None}}, + "body": {"body": 123}, + "null-author": {"author": None}, + "unknown-author": {"author": {"__typename": "Organization", "login": "graphql-org"}}, + "commit-oid": {"commit": {"oid": "not-hex"}}, + } + for name, overrides in shapes.items(): + with self.subTest(shape=name): + node = {"databaseId": 5873000001, "submittedAt": "2026-11-12T12:00:00Z", + "commit": {"oid": PR968_HEAD}, "body": marked_body} + node.update(overrides) + broken = testlib.build_review_node(**node) + snapshot, _ = self._read( + [testlib.build_pr_record(number=968, sha=PR968_HEAD)], + [FakeResponse(200, reviews_page([older_pass, broken]))]) + self.assertEqual(snapshot["outcomes"], {968: "invalid-input"}) + self.assertEqual(snapshot["invalid_input"], ["pr-968"]) + self.assertEqual(snapshot["api_error"], []) + + def test_unrelated_invalid_bot_node_is_ignored(self): + older_pass = testlib.fixture_node("pr968") + unrelated = testlib.build_review_node( + databaseId=5873000002, submittedAt="2026-11-12T14:00:00Z", + commit={"oid": PR968_HEAD}, body="LGTM", state="garbage") + snapshot, _ = self._read( + [testlib.build_pr_record(number=968, sha=PR968_HEAD)], + [FakeResponse(200, reviews_page([older_pass, unrelated]))]) + self.assertEqual(snapshot["outcomes"], {968: "pass"}) + self.assertEqual(snapshot["invalid_input"], []) + + def test_malformed_known_user_review_is_ignored(self): + older_pass = testlib.fixture_node("pr968") + copied_markers = testlib.load_review_fixture("pr985")["body"] + user = testlib.build_review_node( + databaseId=5873000003, submittedAt="whenever", state="garbage", + commit={"oid": PR968_HEAD}, body=copied_markers, + author={"__typename": "User", "login": "graphql-user", "databaseId": 555006}) + snapshot, _ = self._read( + [testlib.build_pr_record(number=968, sha=PR968_HEAD)], + [FakeResponse(200, reviews_page([older_pass, user]))]) + self.assertEqual(snapshot["outcomes"], {968: "pass"}) + self.assertEqual(snapshot["invalid_input"], []) + self.assertEqual(snapshot["api_error"], []) + + def test_malformed_pr_record_flags_the_snapshot_invalid_input(self): + good = testlib.build_pr_record(number=968, sha=PR968_HEAD) + older_pass = testlib.fixture_node("pr968") + shapes = [ + ("number", testlib.build_pr_record(number=0, sha=PR968_HEAD)), + ("head.sha", testlib.build_pr_record(number=7, sha="abc")), + ("head.sha", testlib.build_pr_record(number=7, sha=PR985_HEAD.upper())), + ("base.ref", {"number": 7, "head": {"sha": PR968_HEAD}, "state": "open"}), + ("state", testlib.build_pr_record(number=7, sha=PR968_HEAD, state=None)), + ("record", "not-a-pr-record"), + ] + for token, malformed in shapes: + with self.subTest(token=token, malformed=malformed): + snapshot, transport = self._read( + [malformed, good], + [FakeResponse(200, reviews_page([older_pass]))]) + self.assertIn("pr-record:%s" % token, snapshot["invalid_input"]) + self.assertEqual(snapshot["prs"], [good]) + self.assertEqual(snapshot["outcomes"], {968: "pass"}) + # The malformed record triggers no review read of its own. + self.assertEqual(len(transport.calls), 2) + + def test_only_p_h_members_are_fetched_and_evaluated(self): + other_head = testlib.build_pr_record(number=985, sha=PR985_HEAD) + member = testlib.build_pr_record(number=968, sha=PR968_HEAD) + snapshot, transport = self._read( + [other_head, member], + [FakeResponse(200, reviews_page( + [testlib.fixture_node("pr985", commit={"oid": PR968_HEAD})]))]) + self.assertEqual(snapshot["prs"], [member]) + self.assertEqual(snapshot["outcomes"], {968: "fail"}) + self.assertEqual(sorted(snapshot["reviews"]), [968]) + self.assertEqual(len(transport.calls), 2) + self.assertEqual(json.loads(transport.calls[1].data)["variables"]["number"], 968) + + def test_empty_p_h_reads_cleanly(self): + other_head = testlib.build_pr_record(number=985, sha=PR985_HEAD) + snapshot, transport = self._read([other_head], []) + self.assertEqual(snapshot["prs"], []) + self.assertEqual(snapshot["reviews"], {}) + self.assertEqual(snapshot["outcomes"], {}) + self.assertEqual(snapshot["invalid_input"], []) + self.assertEqual(snapshot["api_error"], []) + self.assertEqual(len(transport.calls), 1) + + def test_pulls_failure_flags_api_error_without_review_reads(self): + transport = testlib.FakeTransport([FakeResponse(500)] * 3) + snapshot = gate.read_sha_snapshot(PR968_HEAD, make_client(transport)) + self.assertEqual(snapshot["api_error"], ["pulls"]) + self.assertEqual(snapshot["prs"], []) + self.assertEqual(snapshot["reviews"], {}) + self.assertEqual(snapshot["outcomes"], {}) + self.assertEqual(len(transport.calls), 3) + + def test_review_fetch_failure_flags_that_pr_api_error(self): + first = testlib.build_pr_record(number=968, sha=PR968_HEAD) + second = testlib.build_pr_record(number=969, sha=PR968_HEAD) + snapshot, transport = self._read( + [first, second], + [FakeResponse(500), FakeResponse(500), FakeResponse(500), + FakeResponse(200, reviews_page([testlib.fixture_node("pr968")]))]) + self.assertEqual(snapshot["api_error"], ["pr-968/reviews"]) + self.assertEqual(snapshot["prs"], [first, second]) + self.assertEqual(snapshot["outcomes"], {968: "api-error", 969: "pass"}) + self.assertEqual(sorted(snapshot["reviews"]), [969]) + self.assertEqual(len(transport.calls), 5) + + +def resolve_env(event, **overrides): + """The env of one resolve run: the shared client settings, the + pull_request_target selector and, for workflow_run, its four + workflow-run selectors.""" + env = { + "GITHUB_EVENT_NAME": event, + "GITHUB_REPOSITORY": REPO, + "GITHUB_API_URL": API, + "GITHUB_GRAPHQL_URL": GRAPHQL, + "GITHUB_TOKEN": GITHUB_TOKEN, + "GITHUB_PR_NUMBER": "985", + } + if event == "workflow_run": + env.update({ + "GITHUB_HEAD_SHA": SHA, + "GITHUB_HEAD_BRANCH": "patch-1", + "GITHUB_HEAD_REPOSITORY": "some-fork/java-profiler", + "GITHUB_PULL_REQUEST_NUMBERS": "[]", + }) + env.update(overrides) + return env + + +def open_main_pr(number, sha, head_repository=REPO, head_branch=None): + """An open-main pulls-list record carrying the selector fields the + resolver compares (head repository full name and head branch).""" + pr = testlib.build_pr_record(number=number, sha=sha) + pr["head"]["ref"] = head_branch or "pr-%d-head" % (number,) + pr["head"]["repo"] = {"full_name": head_repository} + return pr + + +class ResolveCommandTest(unittest.TestCase): + """The resolve subcommand: env-only inputs, per-event SHA selection, + validation, the 256 cap, dedupe and the JSON output.""" + + def _run(self, transport, event, output=None, **overrides): + env = resolve_env(event, **overrides) + if output is not None: + env["GITHUB_OUTPUT"] = output + with contextlib.redirect_stdout(io.StringIO()) as captured: + code = gate.main(["resolve"], env=env, transport=transport) + return code, captured.getvalue() + + def _schedule(self, prs): + return testlib.FakeTransport([FakeResponse(200, prs)]) + + def test_workflow_run_fork_run_with_empty_pr_list_emits_run_head_and_all_open_main_heads(self): + prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] + code, out = self._run(self._schedule(prs), "workflow_run") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD, PR985_HEAD])) + + def test_workflow_run_matches_open_main_prs_by_number(self): + prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] + code, out = self._run(self._schedule(prs), "workflow_run", + GITHUB_PULL_REQUEST_NUMBERS="[968]") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD])) + + def test_workflow_run_matches_by_head_repository_and_branch(self): + prs = [open_main_pr(968, PR968_HEAD, head_repository="some-fork/java-profiler", + head_branch="patch-1"), + open_main_pr(985, PR985_HEAD)] + code, out = self._run(self._schedule(prs), "workflow_run") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD])) + + def test_workflow_run_non_matching_selector_publishes_the_run_head_only(self): + prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] + code, out = self._run(self._schedule(prs), "workflow_run", + GITHUB_PULL_REQUEST_NUMBERS="[7]") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA])) + + def test_workflow_run_number_list_with_null_padding_still_matches(self): + prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] + code, out = self._run(self._schedule(prs), "workflow_run", + GITHUB_PULL_REQUEST_NUMBERS="[968, null]") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD])) + code, out = self._run(self._schedule(prs), "workflow_run", + GITHUB_PULL_REQUEST_NUMBERS="[null]") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD, PR985_HEAD])) + + def test_summary_write_failure_leaves_exit_code_and_stdout_untouched(self): + prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] + with tempfile.TemporaryDirectory() as scratch: + summary = os.path.join(scratch, "missing-dir", "summary.md") + code, out = self._run(self._schedule(prs), "workflow_run", + GITHUB_PULL_REQUEST_NUMBERS="[7]", + GITHUB_STEP_SUMMARY=summary) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA])) + + def test_output_write_failure_exits_one(self): + # The shas output is load-bearing: an I/O failure there fails the + # run like a resolve failure (exit 1), unlike the best-effort + # step summary above. + prs = [open_main_pr(968, PR968_HEAD)] + with tempfile.TemporaryDirectory() as scratch: + output = os.path.join(scratch, "missing-dir", "output.txt") + code, out = self._run(self._schedule(prs), "workflow_run", output=output) + self.assertEqual(code, 1) + self.assertEqual(out, "") + + def test_edited_with_a_base_change_resolves_normally(self): + pr = testlib.build_pr_record(number=985, sha=PR985_HEAD, base_ref="develop") + transport = testlib.FakeTransport([FakeResponse(200, pr)]) + code, out = self._run(transport, "pull_request_target", + GITHUB_PR_ACTION="edited", + GITHUB_CHANGES_BASE_REF="main", + GITHUB_CHANGES_BASE_SHA="b" * 40) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [PR985_HEAD]) + self.assertEqual([call.full_url for call in transport.calls], [PR_URL]) + + def test_edited_without_a_base_change_resolves_to_nothing(self): + # A title/body edit cannot change any decision input: the guard + # returns an empty list with no API call at all. + transport = testlib.FakeTransport([]) + code, out = self._run(transport, "pull_request_target", + GITHUB_PR_ACTION="edited") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), []) + self.assertEqual(transport.calls, []) + + def test_edited_guard_never_trips_on_other_actions(self): + pr = testlib.build_pr_record(number=985, sha=PR985_HEAD, base_ref="develop") + transport = testlib.FakeTransport([FakeResponse(200, pr)]) + code, out = self._run(transport, "pull_request_target", + GITHUB_PR_ACTION="opened") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [PR985_HEAD]) + self.assertEqual([call.full_url for call in transport.calls], [PR_URL]) + + def test_pull_request_target_emits_the_live_head_of_a_closed_retargeted_pr(self): + pr = testlib.build_pr_record(number=985, sha=PR985_HEAD, base_ref="develop", + state="closed") + transport = testlib.FakeTransport([FakeResponse(200, pr)]) + code, out = self._run(transport, "pull_request_target", GITHUB_PR_NUMBER="985") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [PR985_HEAD]) + self.assertEqual([call.full_url for call in transport.calls], [PR_URL]) + + def test_overlapping_selectors_dedupe_to_one_entry(self): + shared = "a" * 40 + prs = [open_main_pr(968, shared), + open_main_pr(969, shared, head_repository="some-fork/java-profiler", + head_branch="patch-1")] + code, out = self._run(self._schedule(prs), "workflow_run", + GITHUB_HEAD_SHA=shared, + GITHUB_PULL_REQUEST_NUMBERS="[968, 969]") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [shared]) + + def test_schedule_and_dispatch_emit_all_open_main_heads(self): + prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] + for event in ("schedule", "workflow_dispatch"): + with self.subTest(event=event): + code, out = self._run(self._schedule(prs), event) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([PR968_HEAD, PR985_HEAD])) + + def test_output_json_shape_via_github_output_and_stdout(self): + with tempfile.TemporaryDirectory() as scratch: + output = os.path.join(scratch, "out.txt") + transport = self._schedule([open_main_pr(968, PR968_HEAD)]) + code, out = self._run(transport, "schedule", output=output) + self.assertEqual(code, 0) + self.assertEqual(out, "") + with open(output, encoding="utf-8") as handle: + written = handle.read() + self.assertEqual(written, "shas=%s\n" % json.dumps([PR968_HEAD])) + transport = self._schedule([open_main_pr(968, PR968_HEAD)]) + code, out = self._run(transport, "schedule") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [PR968_HEAD]) + + def test_more_than_256_shas_exit_1_without_writing_and_256_exit_0(self): + def sha(i): + return "a%039d" % (i,) + + for count, expected in ((256, 0), (257, 1)): + with self.subTest(count=count): + prs = [open_main_pr(1000 + i, sha(i)) for i in range(count)] + with tempfile.TemporaryDirectory() as scratch: + output = os.path.join(scratch, "out.txt") + open(output, "w").close() + code, _ = self._run(self._schedule(prs), "schedule", output=output) + self.assertEqual(code, expected) + with open(output, encoding="utf-8") as handle: + written = handle.read() + if expected == 0: + self.assertEqual(len(json.loads(written.split("=", 1)[1])), 256) + else: + self.assertEqual(written, "") + + def test_invalid_head_sha_env_values_exit_1_before_any_request(self): + for bad in ("1" * 39, "z" * 40, "A" * 40, PR985_HEAD.upper(), + "0x" + "1" * 38, ""): + with self.subTest(bad=bad): + transport = testlib.FakeTransport([]) + code, _ = self._run(transport, "workflow_run", GITHUB_HEAD_SHA=bad) + self.assertEqual(code, 1) + self.assertEqual(transport.calls, []) + + def test_invalid_pr_numbers_exit_1_before_any_request(self): + number_cases = ["0", "-5", "abc", "9.5", None, ""] + for bad in number_cases: + with self.subTest(selector="pull_request_target", bad=bad): + transport = testlib.FakeTransport([]) + code, _ = self._run(transport, "pull_request_target", GITHUB_PR_NUMBER=bad) + self.assertEqual(code, 1) + self.assertEqual(transport.calls, []) + list_cases = ["[0]", '[-1]', '["968"]', "[968, 0]", "[9.5]", "968", + "not json", None, ""] + for bad in list_cases: + with self.subTest(selector="workflow_run list", bad=bad): + transport = testlib.FakeTransport([]) + code, _ = self._run(transport, "workflow_run", GITHUB_PULL_REQUEST_NUMBERS=bad) + self.assertEqual(code, 1) + self.assertEqual(transport.calls, []) + + def test_unknown_event_name_or_missing_env_exits_1(self): + missing = [ + ("push", {}), + ("pull_request_target", {"GITHUB_PR_NUMBER": None}), + ("workflow_run", {"GITHUB_HEAD_SHA": None}), + ("workflow_run", {"GITHUB_HEAD_BRANCH": None}), + ("workflow_run", {"GITHUB_HEAD_REPOSITORY": None}), + ("workflow_run", {"GITHUB_PULL_REQUEST_NUMBERS": None}), + ] + for event, overrides in missing: + with self.subTest(event=event, overrides=sorted(overrides)): + transport = testlib.FakeTransport([]) + code, _ = self._run(transport, event, **overrides) + self.assertEqual(code, 1) + self.assertEqual(transport.calls, []) + for repo in (None, "java-profiler", "DataDog/"): + with self.subTest(repository=repo): + code, _ = self._run(testlib.FakeTransport([]), "schedule", + GITHUB_REPOSITORY=repo) + self.assertEqual(code, 1) + + def test_api_failure_after_retries_exits_1(self): + transport = testlib.FakeTransport([FakeResponse(500)] * 3) + code, _ = self._run(transport, "schedule") + self.assertEqual(code, 1) + self.assertEqual(len(transport.calls), 3) + transport = testlib.FakeTransport([FakeResponse(404)]) + code, _ = self._run(transport, "pull_request_target") + self.assertEqual(code, 1) + self.assertEqual(len(transport.calls), 1) + + def test_malformed_open_main_record_fails_closed_even_when_a_selector_matched(self): + prs = [open_main_pr(968, PR968_HEAD), testlib.build_pr_record(number=7, sha="abc")] + with tempfile.TemporaryDirectory() as scratch: + output = os.path.join(scratch, "out.txt") + open(output, "w").close() + code, _ = self._run(self._schedule(prs), "workflow_run", + GITHUB_PULL_REQUEST_NUMBERS="[968]", output=output) + self.assertEqual(code, 1) + with open(output, encoding="utf-8") as handle: + self.assertEqual(handle.read(), "") + + def test_malformed_fetched_pr_record_exits_1(self): + pr = testlib.build_pr_record(number=985, sha=PR985_HEAD.upper()) + transport = testlib.FakeTransport([FakeResponse(200, pr)]) + code, _ = self._run(transport, "pull_request_target", GITHUB_PR_NUMBER="985") + self.assertEqual(code, 1) + + def test_head_repository_and_branch_values_reach_only_comparisons(self): + prs = [open_main_pr(968, PR968_HEAD)] + transport = self._schedule(prs) + code, out = self._run(transport, "workflow_run", + GITHUB_HEAD_REPOSITORY="evil/repo; rm -rf /", + GITHUB_HEAD_BRANCH="$(touch pwned)", + GITHUB_PULL_REQUEST_NUMBERS="[968]") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD])) + self.assertEqual([call.full_url for call in transport.calls], [PULLS_URL]) + + +class ResolveEnvOnlyContractTest(unittest.TestCase): + """The resolver executes nothing: the gate module carries no command + execution surface.""" + + def test_module_source_has_no_command_execution_surface(self): + with open(gate.__file__, encoding="utf-8") as handle: + source = handle.read() + self.assertNotIn("subprocess", source) + self.assertNotIn("os.system", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_bits_review_gate_policy.py b/.github/scripts/tests/test_bits_review_gate_policy.py new file mode 100644 index 0000000000..e5b4f99b67 --- /dev/null +++ b/.github/scripts/tests/test_bits_review_gate_policy.py @@ -0,0 +1,884 @@ +# Copyright 2026, Datadog, Inc. +"""Scaffold exercises and policy-core tests for the Bits review gate. + +Holds the fixture-, builder- and mapping-behavior checks of the scaffold +plus the policy family's validation, scope, candidacy, outcome, decision +and fingerprint tests; the adapter and CLI families extend this module. +""" + +import os +import re +import sys +import unittest + +# The policy module lives in the parent directory, next to the gate script +# that ships it; discovery starts in this tests directory. +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import bits_review_gate_policy as policy # noqa: E402 + +import bits_review_gate_testlib as testlib + +PR985_HEAD = "1dcc3fa57582f2b595403f2b6a54e4464f352223" +PR968_HEAD = "1115eb583f475b62544d634bb12bf750ae913e04" + + +def fixture_record(pr): + """A validated Bot record carrying a real Bits fixture body.""" + rest = testlib.load_review_fixture(pr) + return testlib.build_review_record( + id=rest["id"], author_id=rest["user"]["id"], author_login=rest["user"]["login"], + state=rest["state"], body=rest["body"], commit_oid=rest["commit_id"], + submitted_at=rest["submitted_at"]) + + +def build_pr(number=985, sha=PR985_HEAD, base_ref="main", state="open"): + """A REST pulls-list shaped PR record.""" + return {"number": number, "head": {"sha": sha}, "base": {"ref": base_ref}, "state": state} + + +REVIEWER_ID_COMMENT = "" +PROD_ENV_COMMENT = "" +_FULL_COMMENT_RE = re.compile(r"", re.DOTALL) + + +def passing_record(**overrides): + """A never-edited, allowlisted, valid PASS record bound to PR 968's head.""" + record = fixture_record("pr968") + record.update(overrides) + return record + + +def pass_body(): + """The real never-edited PASS fixture body.""" + return fixture_record("pr968")["body"] + + +def meta_comment(body): + """The single autotest-meta comment of a body.""" + return [c for c in _FULL_COMMENT_RE.findall(body) if "autotest-meta" in c][0] + + +def drop_meta(body): + return body.replace(meta_comment(body), "") + + +def drop_env(body): + return body.replace(PROD_ENV_COMMENT, "") + + +def snapshot_records(head=PR968_HEAD): + """Two normalized Bot records bound to head, ids 1 and 2.""" + return [ + testlib.build_review_record(id=1, author_id=88084959, commit_oid=head, + body="first plain text", submitted_at="2026-11-12T09:00:00Z"), + testlib.build_review_record(id=2, author_id=214633350, commit_oid=head, + body=pass_body(), submitted_at="2026-11-12T10:00:00Z"), + ] + + +def base_snapshot(head=PR968_HEAD): + """A seam-shaped snapshot: two P(H) members sharing head, their + normalized records and no error flags.""" + return { + "sha": head, + "prs": [testlib.build_pr_record(number=968, sha=head), + testlib.build_pr_record(number=985, sha=head)], + "reviews": {968: snapshot_records(head), + 985: [testlib.build_review_record(id=3, commit_oid=head, body="LGTM")]}, + "api_error": [], + "invalid_input": [], + "outcomes": {968: "pass", 985: "missing"}, + } + + +class FixtureLoadingTest(unittest.TestCase): + """Each fixture file loads and supplies the real REST review values.""" + + def test_pr985_fixture_supplies_real_review_values(self): + rest = testlib.load_review_fixture("pr985") + self.assertEqual(rest["id"], 5872953227) + self.assertEqual(rest["commit_id"], "1dcc3fa57582f2b595403f2b6a54e4464f352223") + self.assertEqual(rest["user"]["id"], 214633350) + self.assertEqual(rest["user"]["type"], "Bot") + self.assertEqual(rest["user"]["login"], "datadog-official[bot]") + self.assertEqual(rest["submitted_at"], "2026-11-12T10:38:09Z") + self.assertEqual(rest["state"], "COMMENTED") + self.assertIn("verdict=FAIL", rest["body"]) + self.assertIn("autotest-meta", rest["body"]) + + def test_pr968_fixture_supplies_real_review_values(self): + rest = testlib.load_review_fixture("pr968") + self.assertEqual(rest["id"], 5872101609) + self.assertEqual(rest["commit_id"], "1115eb583f475b62544d634bb12bf750ae913e04") + self.assertEqual(rest["user"]["id"], 88084959) + self.assertEqual(rest["user"]["type"], "Bot") + self.assertEqual(rest["user"]["login"], "datadog-datadog-prod-us1[bot]") + self.assertEqual(rest["submitted_at"], "2026-11-12T09:30:55Z") + self.assertEqual(rest["state"], "COMMENTED") + self.assertIn("verdict=PASS", rest["body"]) + self.assertIn("autotest-meta", rest["body"]) + + +class BuilderContractTest(unittest.TestCase): + """Builders emit exactly the contract's node and record fields.""" + + def test_node_builder_emits_exactly_contract_node_fields(self): + node = testlib.build_review_node() + self.assertEqual(set(node), {"databaseId", "state", "body", "submittedAt", "lastEditedAt", "commit", "author"}) + self.assertEqual(set(node["commit"]), {"oid"}) + self.assertEqual(set(node["author"]), {"__typename", "login", "databaseId"}) + self.assertEqual(testlib.build_review_node(databaseId=7)["databaseId"], 7) + + def test_record_builder_emits_exactly_contract_record_fields(self): + record = testlib.build_review_record() + self.assertEqual(set(record), {"id", "author_typename", "author_id", "author_login", "state", "body", "commit_oid", "submitted_at", "last_edited_at"}) + self.assertEqual(testlib.build_review_record(state="APPROVED")["state"], "APPROVED") + + +class RestToNodeMappingTest(unittest.TestCase): + """The REST-fixture to GraphQL-node mapping preserves fixture data.""" + + def test_mapping_carries_each_fixture_field(self): + for rest in testlib.load_review_fixtures(): + with self.subTest(id=rest["id"]): + node = testlib.rest_review_to_node(rest) + self.assertEqual(node["databaseId"], rest["id"]) + self.assertEqual(node["state"], rest["state"]) + self.assertEqual(node["submittedAt"], rest["submitted_at"]) + self.assertEqual(node["commit"]["oid"], rest["commit_id"]) + self.assertEqual(node["author"]["databaseId"], rest["user"]["id"]) + self.assertEqual(node["author"]["__typename"], rest["user"]["type"]) + self.assertEqual(node["author"]["login"], rest["user"]["login"]) + + def test_mapping_round_trips_fixture_bodies_byte_identically(self): + for rest in testlib.load_review_fixtures(): + with self.subTest(id=rest["id"]): + node = testlib.rest_review_to_node(rest) + self.assertEqual(node["body"], rest["body"]) + self.assertIn("verdict=", node["body"]) + self.assertIsNone(node["lastEditedAt"]) + + +class MarkedTest(unittest.TestCase): + """The three marker substrings each mark a body on their own.""" + + def test_each_marker_substring_alone_marks_the_body(self): + for marker in ("autotest-meta", "datadog-code-review", "datadog-autotest"): + with self.subTest(marker=marker): + self.assertTrue(policy.is_marked("looks human done" % marker)) + self.assertFalse(policy.is_marked("plain review text")) + self.assertFalse(policy.is_marked(None)) + + +class EditedStateTest(unittest.TestCase): + """Edit state derives from the raw last_edited_at value only.""" + + def test_raw_last_edited_value_is_retained_and_marks_edited(self): + record = testlib.build_review_record(last_edited_at="2026-11-12T11:00:00Z") + self.assertTrue(policy.is_edited(record)) + self.assertEqual(record["last_edited_at"], "2026-11-12T11:00:00Z") + + def test_null_last_edited_is_not_edited(self): + record = testlib.build_review_record() + self.assertFalse(policy.is_edited(record)) + self.assertIsNone(record["last_edited_at"]) + + +class ScopeTest(unittest.TestCase): + """Scope binds to the exact 40-lowercase-hex head only.""" + + def test_fixture_record_is_in_scope_for_its_own_head(self): + self.assertTrue(policy.is_in_scope(fixture_record("pr985"), PR985_HEAD)) + self.assertTrue(policy.is_in_scope(fixture_record("pr968"), PR968_HEAD)) + + def test_head_variants_are_out_of_scope(self): + record = fixture_record("pr985") + for head in (PR985_HEAD[:-1], PR985_HEAD + "a", PR985_HEAD.upper(), "2" + PR985_HEAD[1:], "0" * 40): + with self.subTest(head=head): + self.assertFalse(policy.is_in_scope(record, head)) + + def test_null_commit_pending_and_null_submitted_are_out_of_scope(self): + self.assertFalse(policy.is_in_scope(testlib.build_review_record(commit_oid=None), PR985_HEAD)) + self.assertFalse(policy.is_in_scope(testlib.build_review_record(commit_oid=PR985_HEAD, state="PENDING"), PR985_HEAD)) + self.assertFalse(policy.is_in_scope(testlib.build_review_record(commit_oid=PR985_HEAD, submitted_at=None), PR985_HEAD)) + +class CandidateTest(unittest.TestCase): + """Candidates are exactly the marked or edited Bot reviews.""" + + def test_marked_and_edited_bot_reviews_are_candidates(self): + marked = testlib.build_review_record(body=fixture_record("pr968")["body"]) + edited = testlib.build_review_record(last_edited_at="2026-11-12T11:00:00Z") + self.assertTrue(policy.is_candidate(marked)) + self.assertTrue(policy.is_candidate(edited)) + self.assertTrue(policy.is_candidate(fixture_record("pr985"))) + + def test_user_copied_markers_are_not_candidates(self): + record = fixture_record("pr985") + record["author_typename"] = "User" + self.assertTrue(policy.is_marked(record["body"])) + self.assertFalse(policy.is_candidate(record)) + + def test_allowlisted_unedited_unmarked_bot_is_not_a_candidate(self): + record = testlib.build_review_record(author_id=214633350, author_login="datadog-official[bot]", body="LGTM") + self.assertFalse(policy.is_candidate(record)) + + +class ReviewValidationFailureTest(unittest.TestCase): + """Every contract failure shape fails with its own field token.""" + + def test_invalid_shapes_fail_with_their_token(self): + marked_body = fixture_record("pr985")["body"] + cases = [ + ("body", {"body": 123}), ("last_edited_at", {"last_edited_at": "not-a-timestamp"}), + ("last_edited_at", {"last_edited_at": 17}), ("id", {"id": 0}), + ("id", {"id": -3}), ("id", {"id": "5872953227"}), + ("id", {"id": True}), ("state", {"state": "CHANGED_REQUESTED"}), + ("state", {"state": "open"}), ("state", {"state": None}), + ("commit_oid", {"commit_oid": PR985_HEAD[:-1]}), ("commit_oid", {"commit_oid": PR985_HEAD.upper()}), + ("commit_oid", {"commit_oid": "z" * 40}), ("submitted_at", {"submitted_at": None}), + ("submitted_at", {"submitted_at": "garbage"}), ("author_id", {"author_id": 0}), + ("author_id", {"author_id": None}), + ("author_typename", {"author_typename": None, "body": marked_body}), + ("author_typename", {"author_typename": "Organization", "body": marked_body}), + ("author_typename", {"author_typename": "Mannequin", "last_edited_at": "2026-11-12T11:00:00Z"}), + ] + for reason, overrides in cases: + with self.subTest(reason=reason, overrides=overrides): + record = testlib.build_review_record(**overrides) + self.assertEqual(policy.validate_review_record(record), reason) + + def test_user_record_is_ignored_without_further_checks(self): + record = testlib.build_review_record( + author_typename="User", author_id=0, body=123, id=-1, + state="garbage", commit_oid="zz", submitted_at="nope") + self.assertIsNone(policy.validate_review_record(record)) + + def test_unmarked_null_author_record_is_not_a_validation_failure(self): + record = testlib.build_review_record(author_typename=None, author_id=None, body="plain text") + self.assertIsNone(policy.validate_review_record(record)) + + +class ReviewValidationPassTest(unittest.TestCase): + """Valid records pass, including both timestamp spellings.""" + + def test_every_review_state_is_valid(self): + for state in policy.REVIEW_STATES: + with self.subTest(state=state): + record = testlib.build_review_record( + state=state, + submitted_at=None if state == "PENDING" else "2026-11-12T09:30:55Z") + self.assertIsNone(policy.validate_review_record(record)) + + def test_offset_and_pending_with_submitted_at_are_valid(self): + offset = testlib.build_review_record(submitted_at="2026-11-12T11:30:55+02:00") + pending = testlib.build_review_record(state="PENDING") + self.assertIsNone(policy.validate_review_record(offset)) + self.assertIsNone(policy.validate_review_record(pending)) + + def test_fixture_derived_records_are_valid_marked_candidates(self): + for pr in ("pr985", "pr968"): + with self.subTest(pr=pr): + record = fixture_record(pr) + self.assertIsNone(policy.validate_review_record(record)) + self.assertTrue(policy.is_marked(record["body"])) + self.assertTrue(policy.is_candidate(record)) + self.assertFalse(policy.is_edited(record)) + + +class PrValidationTest(unittest.TestCase): + """PR records must carry the contract's required fields.""" + + def test_valid_pr_record_passes(self): + self.assertIsNone(policy.validate_pr_record(build_pr())) + self.assertIsNone(policy.validate_pr_record(build_pr(number=968, sha=PR968_HEAD, state="open"))) + + def test_pr_field_failures(self): + cases = [ + ("number", build_pr(number=0)), ("number", build_pr(number=-1)), + ("number", build_pr(number="985")), ("number", build_pr(number=True)), + ("head.sha", build_pr(sha=PR985_HEAD[:-1])), ("head.sha", build_pr(sha=PR985_HEAD.upper())), + ("head.sha", build_pr(sha=None)), + ("head.sha", {"number": 985, "base": {"ref": "main"}, "state": "open"}), + ("base.ref", build_pr(base_ref=7)), + ("base.ref", {"number": 985, "head": {"sha": PR985_HEAD}, "state": "open"}), + ("state", build_pr(state=None)), ("state", build_pr(state=5)), + ] + for reason, pr in cases: + with self.subTest(reason=reason, pr=pr): + self.assertEqual(policy.validate_pr_record(pr), reason) + + +class LatestSelectionTest(unittest.TestCase): + """Latest selection orders by parsed UTC instant, then id, and never + consults dismissal, origin, edit or marker state.""" + + def test_equal_timestamps_break_the_tie_by_the_higher_id(self): + fail = passing_record(id=1, body=fixture_record("pr985")["body"]) + newer_pass = passing_record(id=2) + self.assertEqual(policy.outcome([fail, newer_pass], head=PR968_HEAD), "pass") + self.assertEqual(policy.outcome([newer_pass, fail], head=PR968_HEAD), "pass") + self.assertIs(policy.latest_candidate([fail, newer_pass], PR968_HEAD), newer_pass) + + def test_shuffled_input_keeps_the_selection(self): + oldest = passing_record(id=1, submitted_at="2026-11-12T08:00:00Z") + middle = passing_record(id=2, submitted_at="2026-11-12T09:00:00Z", state="DISMISSED") + newest = passing_record(id=3, submitted_at="2026-11-12T10:00:00Z") + for order in ([oldest, middle, newest], [newest, middle, oldest], + [middle, newest, oldest], [newest, oldest, middle]): + with self.subTest(order=[r["id"] for r in order]): + self.assertEqual(policy.outcome(order, head=PR968_HEAD), "pass") + self.assertIs(policy.latest_candidate(order, PR968_HEAD), newest) + + def test_invalid_id_ties_break_deterministically(self): + # Both records stay marked candidates with equal submitted_at; only + # their ids are invalid, so the canonical JSON member form decides. + first = passing_record(id="985-bad") + second = passing_record(id=None) + # None serializes after "985-bad" in the canonical member JSON, so + # the second record is the deterministic winner. + reference = policy.latest_candidate([first, second], PR968_HEAD) + self.assertIsNotNone(reference) + self.assertIs(reference, second) + for order in ([first, second], [second, first]): + with self.subTest(order=[repr(r["id"]) for r in order]): + self.assertIs(policy.latest_candidate(order, PR968_HEAD), reference) + + def test_equivalent_timestamp_spellings_compare_as_utc_instants(self): + # 09:00+05:00 is 04:00 UTC: lexically greater, instant-earlier. + lexical_trap = passing_record( + id=5, submitted_at="2026-11-12T09:00:00+05:00", body=fixture_record("pr985")["body"]) + half_past = passing_record(id=2, submitted_at="2026-11-12T04:30:00Z") + self.assertEqual(policy.outcome([lexical_trap, half_past], head=PR968_HEAD), "pass") + z = passing_record(id=3, submitted_at="2026-11-12T10:00:00Z", body=fixture_record("pr985")["body"]) + offset = passing_record(id=4, submitted_at="2026-11-12T12:00:00+02:00") + naive = passing_record(id=1, submitted_at="2026-11-12T10:00:00") + self.assertEqual(policy.outcome([z, offset, naive], head=PR968_HEAD), "pass") + self.assertIs(policy.latest_candidate([z, offset, naive], PR968_HEAD), offset) + + def test_an_edit_to_an_older_review_does_not_reorder_it(self): + older_edited = passing_record( + id=1, submitted_at="2026-11-12T08:00:00Z", last_edited_at="2026-11-12T13:00:00Z") + newer_pass = passing_record(id=2) + self.assertEqual(policy.outcome([older_edited, newer_pass], head=PR968_HEAD), "pass") + self.assertIs(policy.latest_candidate([older_edited, newer_pass], PR968_HEAD), newer_pass) + + def test_out_of_scope_records_never_win_the_selection(self): + other_head = passing_record(id=7, submitted_at="2026-11-12T15:00:00Z", commit_oid=PR985_HEAD) + pending = passing_record(id=8, submitted_at="2026-11-12T15:30:00Z", state="PENDING") + null_commit = passing_record(id=9, submitted_at="2026-11-12T16:00:00Z", commit_oid=None) + valid_pass = passing_record(id=2) + records = [other_head, pending, null_commit, valid_pass] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "pass") + self.assertIs(policy.latest_candidate(records, PR968_HEAD), valid_pass) + + +class BindingReasonsTest(unittest.TestCase): + """The contract's binding expected-reasons table.""" + + def test_unlisted_bot_edited_markerless_is_untrusted_origin(self): + record = passing_record(author_id=421337, body="plain bot text", + last_edited_at="2026-11-12T11:00:00Z") + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "untrusted-origin") + + def test_unlisted_bot_marked_pass_is_untrusted_origin(self): + record = passing_record(author_id=421337) + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "untrusted-origin") + + def test_allowlisted_edited_markerless_is_edited_summary(self): + record = passing_record(body="plain bot text", last_edited_at="2026-11-12T11:00:00Z") + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "edited-summary") + + def test_allowlisted_edited_valid_pass_body_is_edited_summary(self): + record = passing_record(last_edited_at="2026-11-12T11:00:00Z") + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "edited-summary") + + def test_allowlisted_dismissed_and_edited_is_dismissed(self): + record = passing_record(state="DISMISSED", last_edited_at="2026-11-12T11:00:00Z") + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "dismissed") + + def test_each_of_the_eight_allowlisted_ids_passes_by_outcome(self): + for author_id in (214633350, 88084959, 261164178, 298424163, + 266080212, 266080418, 266081015, 266788760): + with self.subTest(author_id=author_id): + record = passing_record(author_id=author_id) + self.assertIsNone(record["last_edited_at"]) + self.assertEqual(policy.outcome([record], head=PR968_HEAD), "pass") + adjacent = passing_record(author_id=214633351) + self.assertEqual(policy.outcome([adjacent], head=PR968_HEAD), "untrusted-origin") + + +class NoFallbackTest(unittest.TestCase): + """A bad latest candidate never uncovers an older PASS.""" + + def setUp(self): + self.older_pass = passing_record(id=1, submitted_at="2026-11-12T08:00:00Z") + + def _latest(self, **overrides): + return passing_record(id=2, submitted_at="2026-11-12T12:00:00Z", **overrides) + + def test_latest_dismissed_does_not_fall_back(self): + records = [self.older_pass, self._latest(state="DISMISSED")] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "dismissed") + + def test_latest_edited_does_not_fall_back(self): + records = [self.older_pass, self._latest(last_edited_at="2026-11-12T13:00:00Z")] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "edited-summary") + + def test_latest_malformed_does_not_fall_back(self): + records = [self.older_pass, self._latest(body=drop_meta(pass_body()))] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "malformed:missing-meta") + + def test_latest_untrusted_does_not_fall_back(self): + records = [self.older_pass, self._latest(author_id=421337)] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "untrusted-origin") + + def test_a_newer_never_edited_pass_clears_an_older_edited_block(self): + older_edited = passing_record( + id=1, submitted_at="2026-11-12T08:00:00Z", last_edited_at="2026-11-12T13:00:00Z") + self.assertEqual(policy.outcome([older_edited, self._latest()], head=PR968_HEAD), "pass") + + +class MalformedMarkerTest(unittest.TestCase): + """Every malformed: reason is reached from the real PASS body.""" + + def _outcome_for_body(self, body): + self.assertEqual(policy.outcome([passing_record(body=body)], head=PR968_HEAD), "malformed:%s" % self.sub_reason) + + def test_missing_meta(self): + self.sub_reason = "missing-meta" + self._outcome_for_body(drop_meta(pass_body())) + two_metes = pass_body() + "\n" + self._outcome_for_body(two_metes) + + def test_conflicting_verdict(self): + self.sub_reason = "conflicting-verdict" + self._outcome_for_body(pass_body().replace("verdict=PASS", "verdict=PASS verdict=FAIL", 1)) + + def test_unknown_verdict(self): + self.sub_reason = "unknown-verdict" + for replacement in ("verdict=WIBBLE", "verdict=pass", "verdict=FAIL,", "verdict="): + with self.subTest(replacement=replacement): + self._outcome_for_body(pass_body().replace("verdict=PASS", replacement, 1)) + self._outcome_for_body(pass_body().replace("verdict=PASS ", "", 1)) + + def test_reviewer_id(self): + self.sub_reason = "reviewer-id" + self._outcome_for_body(pass_body().replace(REVIEWER_ID_COMMENT, "")) + self._outcome_for_body(pass_body() + "\n" + REVIEWER_ID_COMMENT) + wrong_id = REVIEWER_ID_COMMENT.replace("bad90057", "bad90058") + self._outcome_for_body(pass_body().replace(REVIEWER_ID_COMMENT, wrong_id)) + + def test_env_marker(self): + self.sub_reason = "env-marker" + self._outcome_for_body(drop_env(pass_body())) + staging = pass_body().replace(PROD_ENV_COMMENT, "") + self._outcome_for_body(staging) + self._outcome_for_body(pass_body() + "\n" + PROD_ENV_COMMENT) + + def test_sub_reasons_follow_the_contract_order(self): + # meta before verdict: both wrong reports missing-meta + self.sub_reason = "missing-meta" + self._outcome_for_body(drop_env(drop_meta(pass_body()))) + # verdict before reviewer-id + self.sub_reason = "unknown-verdict" + self._outcome_for_body( + pass_body().replace("verdict=PASS", "verdict=MAYBE", 1).replace(REVIEWER_ID_COMMENT, "")) + # reviewer-id before env + self.sub_reason = "reviewer-id" + self._outcome_for_body( + pass_body().replace(REVIEWER_ID_COMMENT, "").replace(PROD_ENV_COMMENT, + "")) + + def test_badges_image_text_and_delivery_markers_are_ignored(self): + badge_flipped = pass_body().replace("BITS%20CODE%20REVIEW-PASS", "BITS%20CODE%20REVIEW-FAIL", 1) + badge_flipped = badge_flipped.replace("Bits Code Review: PASS", "Bits Code Review: FAIL", 1) + self._assert_pass(badge_flipped) + delivery_retagged = pass_body().replace( + "datadog-autotest-delivery:v1:prod:", "datadog-autotest-delivery:v1:staging:", 1) + self._assert_pass(delivery_retagged) + + def _assert_pass(self, body): + self.assertIn("verdict=PASS", meta_comment(body)) + self.assertEqual(policy.outcome([passing_record(body=body)], head=PR968_HEAD), "pass") + + +class InvalidInputPrecedenceTest(unittest.TestCase): + """Rule 1: a possible candidate that failed validation wins wherever it + sits in the order, and unrelated invalid reviews never block.""" + + def test_invalid_possible_candidate_at_any_position(self): + valid_pass = passing_record(id=2, submitted_at="2026-11-12T12:00:00Z") + oldest = passing_record(id=1, submitted_at="2026-11-12T08:00:00Z", state="garbage") + middle = passing_record(id=3, submitted_at="2026-11-12T10:00:00Z", body=123) + newest = passing_record(id=4, submitted_at="2026-11-12T15:00:00Z", + last_edited_at="whenever") + broken_by_position = {"oldest": oldest, "middle": middle, "newest": newest} + for name, broken in broken_by_position.items(): + with self.subTest(position=name): + self.assertEqual(policy.outcome([valid_pass, broken], head=PR968_HEAD), "invalid-input") + self.assertEqual(policy.outcome([oldest, middle, newest], head=PR968_HEAD), "invalid-input") + + def test_invalid_input_beats_every_later_rule(self): + latest_edited_untrusted = passing_record( + id=9, submitted_at="2026-11-12T16:00:00Z", author_id=421337, body="plain", + last_edited_at="2026-11-12T16:30:00Z") + broken = passing_record(id=1, submitted_at="2026-11-12T07:00:00Z", + last_edited_at="not-parseable") + records = [latest_edited_untrusted, broken] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "invalid-input") + + def test_unrelated_invalid_bot_reviews_do_not_block(self): + valid_pass = passing_record(id=2, submitted_at="2026-11-12T12:00:00Z") + unrelated = passing_record(id=5, submitted_at="2026-11-12T15:00:00Z", + body="LGTM", state="garbage") + unmarked_null_author = passing_record( + id=6, submitted_at="2026-11-12T15:30:00Z", body="LGTM", + author_typename=None, author_id=None) + records = [valid_pass, unrelated, unmarked_null_author] + self.assertEqual(policy.outcome(records, head=PR968_HEAD), "pass") + + def test_api_error_is_reported_before_any_rule(self): + record = passing_record() + broken = passing_record(id=1, body=123) + self.assertEqual(policy.outcome([record, broken], head=PR968_HEAD, api_error=True), "api-error") + self.assertEqual(policy.outcome([], head=PR968_HEAD, api_error=True), "api-error") + + +class MissingOutcomeTest(unittest.TestCase): + """Rule 2: no in-scope candidate means missing.""" + + def test_empty_and_non_candidate_records_are_missing(self): + self.assertEqual(policy.outcome([], head=PR968_HEAD), "missing") + self.assertEqual(policy.outcome([passing_record(body="LGTM")], head=PR968_HEAD), "missing") + self.assertEqual(policy.outcome([passing_record(state="PENDING")], head=PR968_HEAD), "missing") + self.assertEqual(policy.outcome([passing_record(commit_oid=None)], head=PR968_HEAD), "missing") + + def test_wrong_head_shapes_are_missing(self): + for head in (PR968_HEAD[:-1], PR968_HEAD.upper(), "2" + PR968_HEAD[1:], PR985_HEAD, None): + with self.subTest(head=head): + self.assertEqual(policy.outcome([passing_record()], head=head), "missing") + + def test_none_head_scopes_selection_to_empty(self): + self.assertIsNone(policy.latest_candidate([passing_record()], head=None)) + + +class FixtureOutcomeTest(unittest.TestCase): + """The two supplied fixture reviews yield their binding outcomes.""" + + def test_pr985_fails_and_pr968_passes_on_their_own_heads(self): + self.assertEqual(policy.outcome([fixture_record("pr985")], head=PR985_HEAD), "fail") + self.assertEqual(policy.outcome([fixture_record("pr968")], head=PR968_HEAD), "pass") + + def test_each_fixture_is_missing_for_the_other_head(self): + self.assertEqual(policy.outcome([fixture_record("pr985")], head=PR968_HEAD), "missing") + self.assertEqual(policy.outcome([fixture_record("pr968")], head=PR985_HEAD), "missing") + + +class OutcomeStateMappingTest(unittest.TestCase): + """The contract's outcome-to-state mapping, verbatim.""" + + def test_every_outcome_token_maps_to_its_state(self): + cases = [ + ("pass", "success"), ("fail", "failure"), ("dismissed", "failure"), + ("untrusted-origin", "failure"), ("edited-summary", "failure"), + ("missing", "pending"), ("api-error", "error"), ("invalid-input", "error"), + ] + for token, state in cases: + with self.subTest(token=token): + self.assertEqual(policy.outcome_state(token), state) + + def test_every_malformed_sub_reason_is_failure(self): + for token in ("malformed:missing-meta", "malformed:conflicting-verdict", + "malformed:unknown-verdict", "malformed:reviewer-id", + "malformed:env-marker", "malformed:anything-else"): + with self.subTest(token=token): + self.assertEqual(policy.outcome_state(token), "failure") + + def test_unknown_outcome_token_is_a_caller_bug(self): + for token in ("garbage", "PASS", "", "malformed", None, 5): + with self.subTest(token=token): + with self.assertRaises(ValueError): + policy.outcome_state(token) + + +class DecideTest(unittest.TestCase): + """The per-SHA conjunction: independent per-PR outcomes, severity + error > failure > pending > success, lowest PR number on ties.""" + + def test_pass_and_missing_is_pending_naming_the_pr_without_a_review(self): + decision = policy.decide({968: "pass", 985: "missing"}) + self.assertEqual(decision, {"state": "pending", "reason": "missing", "governing_pr": 985}) + + def test_pass_and_fail_is_failure_governed_by_the_failing_pr(self): + decision = policy.decide({968: "pass", 985: "fail"}) + self.assertEqual(decision, {"state": "failure", "reason": "fail", "governing_pr": 985}) + + def test_failure_outranks_pending_across_prs(self): + decision = policy.decide({968: "fail", 985: "missing"}) + self.assertEqual(decision, {"state": "failure", "reason": "fail", "governing_pr": 968}) + mirrored = policy.decide({968: "missing", 985: "fail"}) + self.assertEqual(mirrored, {"state": "failure", "reason": "fail", "governing_pr": 985}) + + def test_invalid_input_or_api_error_anywhere_is_error(self): + for token in ("invalid-input", "api-error"): + with self.subTest(token=token): + decision = policy.decide({968: "pass", 985: token}) + self.assertEqual(decision["state"], "error") + self.assertEqual(decision["reason"], token) + # error dominates failure as well as pending and success + dominating = policy.decide({5: "fail", 7: token}) + self.assertEqual(dominating, + {"state": "error", "reason": token, "governing_pr": 7}) + + def test_success_happens_only_on_the_all_pass_branch(self): + self.assertEqual(policy.decide({5: "pass", 7: "pass", 9: "pass"}), + {"state": "success", "reason": "pass", "governing_pr": 5}) + for token in ("fail", "dismissed", "untrusted-origin", "edited-summary", + "malformed:missing-meta", "missing", "api-error", "invalid-input"): + with self.subTest(token=token): + decision = policy.decide({5: "pass", 7: token}) + self.assertNotEqual(decision["state"], "success") + self.assertEqual(decision["state"], policy.outcome_state(token)) + + def test_equal_severity_is_governed_by_the_lowest_pr_number(self): + decision = policy.decide({10: "fail", 3: "edited-summary"}) + self.assertEqual(decision, {"state": "failure", "reason": "edited-summary", "governing_pr": 3}) + both_missing = policy.decide({12: "missing", 4: "missing"}) + self.assertEqual(both_missing, {"state": "pending", "reason": "missing", "governing_pr": 4}) + both_pass = policy.decide({12: "pass", 4: "pass"}) + self.assertEqual(both_pass["governing_pr"], 4) + + def test_each_pr_is_judged_on_its_own_reviews_only(self): + head = PR968_HEAD + pr968_records = [fixture_record("pr968")] + pr985_fail_record = fixture_record("pr985") + pr985_fail_record["commit_oid"] = head + pr985_fail = [pr985_fail_record] + self.assertEqual(policy.outcome(pr968_records, head=head), "pass") + self.assertEqual(policy.outcome(pr985_fail, head=head), "fail") + decision = policy.decide({968: "pass", 985: "fail"}) + self.assertEqual( + decision, {"state": "failure", "reason": "fail", "governing_pr": 985}) + # a PASS on one shared-head PR never clears another PR's block + pending = policy.decide({968: "pass", 985: "missing"}) + self.assertEqual(pending["state"], "pending") + self.assertEqual(pending["governing_pr"], 985) + + def test_an_empty_p_h_is_never_a_decision(self): + self.assertIsNone(policy.decide({})) + + def test_caller_contract_violations_raise(self): + for bad in ({0: "pass"}, {"5": "pass"}, {True: "pass"}, {1: "garbage"}): + with self.subTest(per_pr_outcomes=bad): + with self.assertRaises(ValueError): + policy.decide(bad) + + +class DescribeTest(unittest.TestCase): + """`Bits review on PR #[: ]`, capped at 140.""" + + def test_description_shape_with_and_without_detail(self): + self.assertEqual(policy.describe("missing", 985), "Bits review missing on PR #985") + self.assertEqual(policy.describe("malformed:missing-meta", 968), + "Bits review malformed:missing-meta on PR #968") + self.assertEqual(policy.describe("api-error", 12, "pulls"), + "Bits review api-error on PR #12: pulls") + self.assertEqual(policy.describe("pass", 985, ""), "Bits review pass on PR #985") + + def test_description_is_capped_at_140_characters(self): + long_detail = "d" * 300 + capped = policy.describe("missing", 985, long_detail) + self.assertEqual(len(capped), 140) + self.assertTrue(capped.startswith("Bits review missing on PR #985: ")) + huge_reason = "r" * 200 + self.assertEqual(len(policy.describe(huge_reason, 985)), 140) + self.assertEqual(len(policy.describe("fail", 1, "x" * 132)), 140) + + def test_caller_contract_violations_raise(self): + for bad_reason, bad_number, bad_detail in ( + (123, 985, None), (None, 985, None), ("pass", 0, None), + ("pass", "985", None), ("pass", -1, None), ("pass", 985, 7)): + with self.subTest(bad=(bad_reason, bad_number, bad_detail)): + with self.assertRaises(ValueError): + policy.describe(bad_reason, bad_number, bad_detail) + + +class TargetUrlTest(unittest.TestCase): + """The governing review's URL when a review governs, else the PR URL; + both are built from API numbers only.""" + + def test_both_url_shapes(self): + review_url = policy.target_url("DataDog/java-profiler", 968, 5872101609) + self.assertEqual(review_url, + "https://github.com/DataDog/java-profiler/pull/968#pullrequestreview-5872101609") + pr_url = policy.target_url("DataDog/java-profiler", 968) + self.assertEqual(pr_url, "https://github.com/DataDog/java-profiler/pull/968") + self.assertNotIn("pullrequestreview", pr_url) + + def test_urls_are_built_from_api_numbers_only(self): + self.assertEqual(policy.target_url("o/p", 7, 9), + "https://github.com/o/p/pull/7#pullrequestreview-9") + self.assertEqual(policy.STATUS_CONTEXT, "bits-review/head") + + def test_caller_contract_violations_raise(self): + for repo, number, review_id in ( + ("", 968, None), (None, 968, None), ("o/p", 0, None), + ("o/p", "968", None), ("o/p", 968, 0), ("o/p", 968, -1), + ("o/p", 968, "5872101609")): + with self.subTest(args=(repo, number, review_id)): + with self.assertRaises(ValueError): + policy.target_url(repo, number, review_id) + + +class FingerprintStabilityTest(unittest.TestCase): + """The fingerprint is byte-stable when nothing changed and canonical: + input ordering and derived outcomes never reach the digest.""" + + def test_identical_snapshots_hash_identically(self): + self.assertEqual(policy.fingerprint(base_snapshot()), policy.fingerprint(base_snapshot())) + first, second = base_snapshot(), base_snapshot() + self.assertIsNot(first, second) + self.assertEqual(policy.fingerprint(first), policy.fingerprint(second)) + + def test_record_and_member_ordering_never_reaches_the_digest(self): + base = base_snapshot() + reordered = base_snapshot() + reordered["reviews"][968] = list(reversed(reordered["reviews"][968])) + reordered["prs"] = list(reversed(reordered["prs"])) + self.assertNotEqual(base["prs"][0]["number"], reordered["prs"][0]["number"]) + self.assertEqual(policy.fingerprint(base), policy.fingerprint(reordered)) + + def test_invalid_entry_ordering_never_reaches_the_digest(self): + # Invalid PR numbers and record ids share one sort bucket; the + # bucket must canonically self-order so input order never reaches + # the digest. + base = base_snapshot() + base["reviews"][985] = [ + testlib.build_review_record(id="bad-two", commit_oid=PR968_HEAD, body="LGTM two"), + testlib.build_review_record(id=None, commit_oid=PR968_HEAD, body="LGTM one"), + ] + base["prs"] = [ + testlib.build_pr_record(number=None, sha=PR968_HEAD), + testlib.build_pr_record(number="not-a-number", sha=PR985_HEAD), + ] + reordered = base_snapshot() + reordered["reviews"][985] = list(reversed(base["reviews"][985])) + reordered["prs"] = list(reversed(base["prs"])) + self.assertEqual(policy.fingerprint(base), policy.fingerprint(reordered)) + + def test_derived_outcomes_never_reach_the_digest(self): + base = base_snapshot() + changed_outcomes = base_snapshot() + changed_outcomes["outcomes"] = {968: "fail", 985: "dismissed"} + self.assertEqual(policy.fingerprint(base), policy.fingerprint(changed_outcomes)) + + def test_unrelated_pr_fields_never_reach_the_digest(self): + base = base_snapshot() + noisy = base_snapshot() + noisy["prs"][0]["title"] = "a totally different title" + noisy["reviews"][985][0]["extra"] = "unrelated field" + self.assertEqual(policy.fingerprint(base), policy.fingerprint(noisy)) + +class FingerprintInputCoverageTest(unittest.TestCase): + """Every decision-input class changes the digest: a decision computed + on an earlier snapshot is never reused after any of them differs.""" + + def _assert_changes(self, mutate): + before = policy.fingerprint(base_snapshot()) + snapshot = base_snapshot() + mutate(snapshot) + self.assertNotEqual(policy.fingerprint(snapshot), before) + + def test_last_edited_at_null_to_value_and_value_to_value(self): + self._assert_changes(lambda s: s["reviews"][968][0].update(last_edited_at="2026-11-12T11:00:00Z")) + self._assert_changes(lambda s: s["reviews"][968][1].update(last_edited_at="2026-11-13T11:00:00Z")) + edited = base_snapshot() + edited["reviews"][985][0]["last_edited_at"] = "2026-11-12T11:00:00Z" + value_fp = policy.fingerprint(edited) + self.assertNotEqual(value_fp, policy.fingerprint(base_snapshot())) + edited["reviews"][985][0]["last_edited_at"] = "2026-11-14T11:00:00Z" + self.assertNotEqual(policy.fingerprint(edited), value_fp) + + def test_author_fields(self): + self._assert_changes(lambda s: s["reviews"][968][0].update(author_id=88084960)) + self._assert_changes(lambda s: s["reviews"][968][0].update(author_typename="User")) + self._assert_changes(lambda s: s["reviews"][968][0].update(author_typename=None)) + self._assert_changes(lambda s: s["reviews"][968][0].update(author_login="other[bot]")) + self._assert_changes(lambda s: s["reviews"][968][0].update(author_id=None)) + + def test_submitted_at_value_and_order(self): + self._assert_changes( + lambda s: s["reviews"][968][0].update(submitted_at="2026-11-12T09:30:00Z")) + # order-relevant: swapping the two records' submitted_at values keeps + # the multiset but changes which review is newest + self._assert_changes( + lambda s: s["reviews"][968][0].update(submitted_at="2026-11-12T10:00:00Z")) + swapped = base_snapshot() + swapped["reviews"][968] = snapshot_records() + swapped["reviews"][968][0]["submitted_at"] = "2026-11-12T10:00:00Z" + swapped["reviews"][968][1]["submitted_at"] = "2026-11-12T09:00:00Z" + self.assertNotEqual(policy.fingerprint(swapped), policy.fingerprint(base_snapshot())) + + def test_state_body_commit_oid_and_record_id(self): + self._assert_changes(lambda s: s["reviews"][968][0].update(state="DISMISSED")) + self._assert_changes(lambda s: s["reviews"][985][0].update(body="LGTN")) + self._assert_changes(lambda s: s["reviews"][985][0].update(body=123)) + self._assert_changes(lambda s: s["reviews"][985][0].update(commit_oid="f" * 40)) + self._assert_changes(lambda s: s["reviews"][985][0].update(commit_oid=None)) + self._assert_changes(lambda s: s["reviews"][985][0].update(id=4)) + + def test_the_snapshot_sha_key_is_not_an_independent_input(self): + # the head is bound by each member's head.sha; the top-level key is + # redundant with it and must not add instability to the digest + relabeled = base_snapshot() + relabeled["sha"] = "0" * 40 + self.assertEqual(policy.fingerprint(relabeled), policy.fingerprint(base_snapshot())) + + def test_the_members_head_sha_binds_the_digest(self): + self._assert_changes(lambda s: s["prs"][0].update(head={"sha": "f" * 40})) + retargeted = base_snapshot() + retargeted["prs"][1]["head"]["sha"] = "e" * 40 + self.assertNotEqual(policy.fingerprint(retargeted), policy.fingerprint(base_snapshot())) + + def test_pr_membership_base_and_state(self): + self._assert_changes( + lambda s: s["prs"].append(testlib.build_pr_record(number=100, sha=s["sha"]))) + self._assert_changes(lambda s: s["prs"].pop()) + self._assert_changes(lambda s: s["prs"][0]["base"].update(ref="develop")) + self._assert_changes(lambda s: s["prs"][0].update(state="closed")) + self._assert_changes(lambda s: s["prs"][0].update(head={"sha": "f" * 40})) + + def test_error_flags_with_their_locations(self): + self._assert_changes(lambda s: s["api_error"].append("pulls")) + self._assert_changes(lambda s: s["invalid_input"].append("pr-985")) + self._assert_changes(lambda s: s["invalid_input"].append("pr-record:number")) + self._assert_changes(lambda s: s["api_error"].append("pr-968/reviews")) + flagged = base_snapshot() + flagged["api_error"] = ["pulls"] + one_flag = policy.fingerprint(flagged) + self.assertNotEqual(one_flag, policy.fingerprint(base_snapshot())) + flagged["api_error"] = ["pulls", "pr-968/reviews"] + self.assertNotEqual(policy.fingerprint(flagged), one_flag) + + def test_new_and_removed_reviews(self): + self._assert_changes( + lambda s: s["reviews"][985].append( + testlib.build_review_record(id=5, commit_oid=s["sha"], body="late review"))) + self._assert_changes(lambda s: s["reviews"][985].pop()) + self._assert_changes(lambda s: s["reviews"].update({100: []})) + + def test_malformed_prs_shape_changes_the_digest(self): + self._assert_changes(lambda s: s.update(prs=None)) + self._assert_changes(lambda s: s.update(prs="not-a-list")) + # two distinct malformed shapes must not collapse to one digest + none_prs = base_snapshot() + none_prs["prs"] = None + str_prs = base_snapshot() + str_prs["prs"] = "not-a-list" + self.assertNotEqual(policy.fingerprint(none_prs), policy.fingerprint(str_prs)) + # _jsonable output is always JSON-serializable + policy.fingerprint({"prs": None, "reviews": {}, "api_error": [], "invalid_input": []}) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_bits_review_gate_publish.py b/.github/scripts/tests/test_bits_review_gate_publish.py new file mode 100644 index 0000000000..d3307f9a75 --- /dev/null +++ b/.github/scripts/tests/test_bits_review_gate_publish.py @@ -0,0 +1,698 @@ +# Copyright 2026, Datadog, Inc. +"""Validated-decision seam and publish CLI tests for the Bits review gate. + +``revalidated_decision`` runs the real snapshot read through the shared +fake transport: every case asserts the return value and the exact +read-call interleaving, and every case proves zero write attempts — no +call leaves the pulls/GraphQL read URLs and none carries STATUS_TOKEN. +The seam itself never reads or writes commit statuses. + +The publish families below run the real CLI (``main``) over the same +transport: ``--sha`` validation, the seam's exit mapping, suppression +against the selected latest app status, the POST payload and its +STATUS_TOKEN-only authorization, and the exit code of every write +branch. +""" + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import bits_review_gate as gate # noqa: E402 +import bits_review_gate_policy as policy # noqa: E402 + +import bits_review_gate_testlib as testlib # noqa: E402 + +REPO = "DataDog/java-profiler" +API = "https://api.example.test" +GRAPHQL = "https://api.example.test/graphql" +GITHUB_TOKEN = "gh-read-token" +STATUS_TOKEN = "status-write-token" +#: The publisher app login whose statuses are the gate's own output. +STATUS_CREATOR = "dd-octo-sts[bot]" +OTHER_CREATOR = "somebody-else[bot]" +PASS_DESCRIPTION = "Bits review pass on PR #968" +INVALID_INPUT_DESCRIPTION = "Bits review invalid-input on PR #968: pr-968" +#: The head under test; the pr968 fixture PASS binds to exactly this SHA. +HEAD = "1115eb583f475b62544d634bb12bf750ae913e04" +OTHER_HEAD = "1dcc3fa57582f2b595403f2b6a54e4464f352223" +PULLS_URL = "%s/repos/%s/pulls?state=open&base=main&per_page=100" % (API, REPO) +PASS_REVIEW_ID = 5872101609 +FAIL_REVIEW_ID = 5872953227 +UNLISTED_BOT_ID = 123456 +PR968_URL = "https://github.com/%s/pull/968" % (REPO,) +PR969_URL = "https://github.com/%s/pull/969" % (REPO,) +PR985_URL = "https://github.com/%s/pull/985" % (REPO,) +PASS_REVIEW_URL = PR968_URL + "#pullrequestreview-%d" % (PASS_REVIEW_ID,) +FAIL_REVIEW_URL = PR985_URL + "#pullrequestreview-%d" % (FAIL_REVIEW_ID,) + + +class FakeResponse: + """A recorded HTTP response: status, bytes body, optional Link header.""" + + def __init__(self, status, body=b"[]", link=None): + self.status = status + self._body = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") + self.headers = {"Link": link} if link else {} + + def read(self): + return self._body + + +def make_client(transport): + return gate.GitHubClient(REPO, transport=transport, env={ + "GITHUB_API_URL": API, "GITHUB_GRAPHQL_URL": GRAPHQL, + "GITHUB_TOKEN": GITHUB_TOKEN, "STATUS_TOKEN": STATUS_TOKEN}) + + +def publish_env(): + """The publish CLI's environment: the repository, the read API + endpoints, the read token and the status token.""" + return {"GITHUB_REPOSITORY": REPO, "GITHUB_API_URL": API, + "GITHUB_GRAPHQL_URL": GRAPHQL, "GITHUB_TOKEN": GITHUB_TOKEN, + "STATUS_TOKEN": STATUS_TOKEN} + + +def publish_env_without_status_token(): + """The publish environment with the status token removed: the write + cannot authenticate and must fail closed.""" + env = publish_env() + del env["STATUS_TOKEN"] + return env + + +def reviews_page(nodes, has_next=False, cursor="cursor-1"): + return {"data": {"repository": {"pullRequest": {"reviews": { + "nodes": nodes, + "pageInfo": {"hasNextPage": has_next, "endCursor": cursor}}}}}} + + +def failed_reviews_page(): + """A 200 GraphQL response carrying an errors member: fail-closed.""" + page = reviews_page([]) + page["errors"] = [{"message": "Something went wrong"}] + return FakeResponse(200, page) + + +def member(number=968, sha=HEAD, base_ref="main", state="open"): + return testlib.build_pr_record(number=number, sha=sha, base_ref=base_ref, state=state) + + +def pass_node(): + """The pr968 fixture: an unedited allowlisted valid PASS at HEAD.""" + return testlib.fixture_node("pr968") + + +def fail_node(): + """The pr985 fixture FAIL, rebound to the head under test.""" + return testlib.fixture_node("pr985", commit={"oid": HEAD}) + + +def snapshot_script(open_prs, reviews_by_number, head=HEAD): + """(responses, expected read URLs) of one scripted snapshot: the pulls + page, then one reviews page per member (head == the snapshot's head) + in list order. A reviews value may be a prepared response, e.g. a + failed page.""" + responses = [FakeResponse(200, open_prs)] + urls = [PULLS_URL] + for pr in open_prs: + pr_head = pr.get("head") if isinstance(pr, dict) else None + if isinstance(pr_head, dict) and pr_head.get("sha") == head: + value = reviews_by_number[pr["number"]] + responses.append(value if isinstance(value, FakeResponse) + else FakeResponse(200, reviews_page(value))) + urls.append(GRAPHQL) + return responses, urls + + +def failed_pulls_script(): + """(responses, expected read URLs) of one snapshot whose pulls read + fails without a retry (a 4xx response).""" + return [FakeResponse(404)], [PULLS_URL] + + +def status_post_url(sha): + return "%s/repos/%s/statuses/%s" % (API, REPO, sha) + + +def status_record(state="success", description=PASS_DESCRIPTION, + target_url=PASS_REVIEW_URL, context="bits-review/head", + creator=STATUS_CREATOR, created_at="2026-11-12T12:00:00Z", + status_id=100): + """One REST commit-status record, as the statuses endpoint returns + them: the compared fields plus the creator and ordering inputs.""" + return {"state": state, "context": context, "description": description, + "target_url": target_url, "creator": {"login": creator}, + "created_at": created_at, "id": status_id} + + +def pass_snapshot(): + """The stable all-pass snapshot whose decision publishes success.""" + return snapshot_script([member()], {968: [pass_node()]}) + + +def invalid_input_snapshot(): + """The stable snapshot whose member decision is an invalid-input + error.""" + broken = testlib.build_review_node( + databaseId=5873000001, submittedAt="2026-11-12T12:00:00Z", + commit={"oid": HEAD}, body=testlib.load_review_fixture("pr985")["body"], + lastEditedAt="not-a-timestamp") + return snapshot_script([member()], {968: [broken]}) + + +def rate_limited_response(status=429, headers=None): + """A rate-limited response whose honor window (Retry-After 600 s) + exceeds the cap, so the read fails fast with RateLimitError.""" + response = FakeResponse(status, []) + response.headers = {"Retry-After": "600"} if headers is None else headers + return response + + +def statuses_read_url(sha=HEAD): + return "%s/repos/%s/commits/%s/statuses" % (API, REPO, sha) + + +def publish_script(decision_snapshot, status_pages=(), post_response=None): + """The transport queue of one full publish run: the decision's single + snapshot, then every status page, then the POST response when a write + is expected.""" + responses = decision_snapshot[0] + list(status_pages) + if post_response is not None: + responses.append(post_response) + return responses + + +class SeamTestCase(unittest.TestCase): + """Shared runner: the real seam, the exact interleaving assertion and + the zero-write-attempt proof applied to every T7 case — no call leaves + the read URLs (pulls, GraphQL reviews), none reads or writes statuses, + and every call carries only GITHUB_TOKEN, never STATUS_TOKEN.""" + + def run_seam(self, *scripts): + responses, expected = [], [] + for responses_part, urls_part in scripts: + responses.extend(responses_part) + expected.extend(urls_part) + transport = testlib.FakeTransport(responses) + result = gate.revalidated_decision(HEAD, make_client(transport)) + self.assertEqual([call.full_url for call in transport.calls], expected) + for call in transport.calls: + self.assertIn(call.full_url, (PULLS_URL, GRAPHQL)) + self.assertEqual(call.headers.get("Authorization"), "Bearer " + GITHUB_TOKEN) + for value in call.headers.values(): + self.assertNotIn(STATUS_TOKEN, value) + return result, transport + + +class EmptySnapshotTest(SeamTestCase): + """A successfully read empty P(H) returns ("empty", None) and reads + nothing else.""" + + def test_empty_p_h_returns_empty_from_one_snapshot_read(self): + open_prs = [member(969, sha=OTHER_HEAD)] + result, transport = self.run_seam(snapshot_script(open_prs, {})) + self.assertEqual(result, ("empty", None)) + self.assertEqual(len(transport.calls), 1) + + +class StableDecisionTest(SeamTestCase): + """Snapshots return the exact decision payload of the single read; + D is asserted exactly and the one-snapshot read is pinned.""" + + def test_stable_snapshot_returns_the_exact_success_payload(self): + result, _ = self.run_seam(snapshot_script([member()], {968: [pass_node()]})) + self.assertEqual(result[0], "stable") + self.assertEqual(result[1], { + "state": "success", + "reason": "pass", + "governing_pr": 968, + "description": "Bits review pass on PR #968", + "target_url": PASS_REVIEW_URL, + "context": "bits-review/head", + }) + + def test_non_decision_input_fields_do_not_change_the_decision(self): + """PR fields outside the decision inputs (title, draft) change + nothing the decision reads: the payload stays the success.""" + changed = member() + changed["title"] = "renamed after the read" + changed["draft"] = True + result, transport = self.run_seam(snapshot_script([changed], {968: [pass_node()]})) + self.assertEqual(result[0], "stable") + self.assertEqual(result[1]["state"], "success") + self.assertEqual(len(transport.calls), 2) + + def test_stable_fail_payload_names_the_governing_review(self): + result, _ = self.run_seam(snapshot_script([member(985)], {985: [fail_node()]})) + self.assertEqual(result[0], "stable") + self.assertEqual(result[1], { + "state": "failure", + "reason": "fail", + "governing_pr": 985, + "description": "Bits review fail on PR #985", + "target_url": FAIL_REVIEW_URL, + "context": "bits-review/head", + }) + + def test_shared_head_pair_with_one_pass_and_one_missing_is_pending(self): + prs = [member(968), member(969)] + reviews = {968: [pass_node()], 969: []} + result, _ = self.run_seam(snapshot_script(prs, reviews)) + self.assertEqual(result[0], "stable") + self.assertEqual(result[1], { + "state": "pending", + "reason": "missing", + "governing_pr": 969, + "description": "Bits review missing on PR #969", + "target_url": PR969_URL, + "context": "bits-review/head", + }) + + def test_success_requires_every_member_to_pass(self): + both_pass = [member(968), member(969)] + reviews = {968: [pass_node()], + 969: [testlib.fixture_node("pr968", databaseId=PASS_REVIEW_ID + 1)]} + result, _ = self.run_seam(snapshot_script(both_pass, reviews)) + self.assertEqual((result[0], result[1]["state"]), ("stable", "success")) + self.assertEqual(result[1]["governing_pr"], 968) + pass_and_fail = [member(968), member(985)] + reviews = {968: [pass_node()], 985: [fail_node()]} + result, _ = self.run_seam(snapshot_script(pass_and_fail, reviews)) + self.assertEqual(result[1]["state"], "failure") + self.assertNotEqual(result[1]["reason"], "pass") + + +class ErrorDecisionTest(SeamTestCase): + """api-error and invalid-input flags propagate into stable decisions + as error decisions; the stale success is never returned.""" + + def test_invalid_input_review_gives_a_stable_error_decision(self): + marked_body = testlib.load_review_fixture("pr985")["body"] + broken = testlib.build_review_node( + databaseId=5873000001, submittedAt="2026-11-12T12:00:00Z", + commit={"oid": HEAD}, body=marked_body, lastEditedAt="not-a-timestamp") + reviews = {968: [pass_node(), broken]} + result, _ = self.run_seam(snapshot_script([member()], reviews)) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "error", "invalid-input")) + self.assertEqual(result[1]["description"], + "Bits review invalid-input on PR #968: pr-968") + + def test_malformed_pr_record_forces_error_over_clean_members(self): + malformed = testlib.build_pr_record(number=7, sha="not-hex") + prs = [member(), malformed] + reviews = {968: [pass_node()]} + result, transport = self.run_seam(snapshot_script(prs, reviews)) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "error", "invalid-input")) + self.assertEqual(result[1]["governing_pr"], 968) + self.assertEqual(result[1]["description"], + "Bits review invalid-input on PR #968: pr-record:head.sha") + self.assertEqual(result[1]["target_url"], PR968_URL) + self.assertEqual(len(transport.calls), 2) + + def test_failed_pulls_read_gives_a_stable_member_less_error_decision(self): + """No PR could be scoped at all: the decision is still an error — + never empty (the read did not succeed) — and, with no API PR + number in the snapshot, description and target URL stay None.""" + result, transport = self.run_seam(failed_pulls_script()) + self.assertEqual(result[0], "stable") + self.assertEqual(result[1], { + "state": "error", + "reason": "api-error", + "governing_pr": None, + "description": None, + "target_url": None, + "context": "bits-review/head", + }) + self.assertEqual(len(transport.calls), 1) + + def test_pr_record_level_flag_outranks_a_member_api_error_in_reason_attribution(self): + # The reason names the class of the flag that forced the level: a + # malformed pr-record beats a member reviews read failure, and the + # description renders the invalid-input flags, not the member's. + malformed = testlib.build_pr_record(number=7, sha="not-hex") + prs = [member(), malformed] + script = snapshot_script(prs, {968: failed_reviews_page()}) + result, _ = self.run_seam(script) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "error", "invalid-input")) + self.assertEqual(result[1]["governing_pr"], 968) + self.assertEqual(result[1]["description"], + "Bits review invalid-input on PR #968: pr-record:head.sha") + + +class ZeroWriteAttemptTest(SeamTestCase): + """The seam itself never reads or writes commit statuses: it returns + values only, and every transport call stays on the two read URLs.""" + + def test_no_case_ever_touches_the_statuses_endpoint(self): + _, transport = self.run_seam(snapshot_script([member()], {968: [pass_node()]})) + self.assertTrue(transport.calls) + for call in transport.calls: + self.assertNotIn("/statuses", call.full_url) + # Only the GraphQL reviews read may carry a body; a write is a + # data-carrying call to any other URL. + if call.data is not None: + self.assertEqual(call.full_url, GRAPHQL) + + +class PublishTestCase(unittest.TestCase): + """The real publish CLI (``main``) over the shared fake transport.""" + + def run_publish(self, responses, sha=HEAD, env=None): + transport = testlib.FakeTransport(responses) + code = gate.main(["publish", "--sha", sha], + env=env if env is not None else publish_env(), + transport=transport) + return code, transport + + def post_calls(self, transport, sha=HEAD): + return [call for call in transport.calls + if call.full_url == status_post_url(sha)] + + def assert_write_tokens(self, transport, sha=HEAD): + """Every read carries GITHUB_TOKEN and never STATUS_TOKEN; the one + POST carries STATUS_TOKEN.""" + self.assertTrue(transport.calls) + for call in transport.calls: + auth = call.headers.get("Authorization") + if call.full_url == status_post_url(sha): + self.assertEqual(auth, "Bearer " + STATUS_TOKEN) + else: + self.assertEqual(auth, "Bearer " + GITHUB_TOKEN) + + +class PublishShaValidationTest(PublishTestCase): + """--sha is validated before anything is read: an invalid head never + reaches the transport.""" + + def test_invalid_sha_exits_1_without_any_transport_call(self): + invalid = ("", "1" * 39, "1" * 41, "z" * 40, "1" * 39 + "A", + "Z" * 40, "1" * 20 + "g" * 20) + for sha in invalid: + code, transport = self.run_publish([], sha=sha) + self.assertEqual(code, 1) + self.assertEqual(transport.calls, []) + + +class RateLimitPublishTest(PublishTestCase): + """Rate-limited reads failing past the honor cap publish diagnosable + error decisions: the description names the rate-limit flag.""" + + def test_rate_limited_reviews_read_names_the_rate_limit_in_the_description(self): + # A reviews read failing past the honor cap flags the member as + # pr-/reviews:rate-limit; the published description names it. + script = snapshot_script([member()], {968: rate_limited_response()}) + code, transport = self.run_publish(publish_script( + script, [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 1) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + payload = json.loads(posts[0].data) + self.assertEqual(payload["state"], "error") + self.assertEqual(payload["description"], + "Bits review api-error on PR #968: pr-968/reviews:rate-limit") + self.assert_write_tokens(transport) + + def test_rate_limited_pulls_read_still_fails_closed_and_writes(self): + # A pulls read failing past the honor cap must produce a + # member-less error decision, not an empty one: the snapshot-level + # filter matches pulls:rate-limit, so the write is attempted and + # the run exits 1. + script = ([rate_limited_response()], [PULLS_URL]) + code, transport = self.run_publish(publish_script( + script, [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 1) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(json.loads(posts[0].data), + {"state": "error", "context": "bits-review/head"}) + self.assert_write_tokens(transport) + +class SeamExitMappingTest(PublishTestCase): + """The seam's empty return maps to exit 0 with zero write calls — and + no status read at all.""" + + def test_empty_p_h_exits_0_with_no_status_read_or_write(self): + script = snapshot_script([member(969, sha=OTHER_HEAD)], {})[0] + code, transport = self.run_publish(script) + self.assertEqual(code, 0) + self.assertEqual(len(transport.calls), 1) + self.assertEqual(self.post_calls(transport), []) + for call in transport.calls: + self.assertNotIn("/statuses", call.full_url) + + +class SuppressionTest(PublishTestCase): + """Suppression compares only the selected latest app status; an + unchanged decision skips the POST entirely.""" + + def test_unchanged_success_skips_the_post_and_exits_0(self): + code, transport = self.run_publish(publish_script( + pass_snapshot(), [FakeResponse(200, [status_record()])])) + self.assertEqual(code, 0) + self.assertEqual(self.post_calls(transport), []) + self.assertEqual(len(transport.calls), 3) + + def test_unchanged_error_skips_the_post_and_still_exits_1(self): + statuses = FakeResponse(200, [status_record( + state="error", description=INVALID_INPUT_DESCRIPTION, + target_url=PR968_URL)]) + code, transport = self.run_publish(publish_script( + invalid_input_snapshot(), [statuses])) + self.assertEqual(code, 1) + self.assertEqual(self.post_calls(transport), []) + + def test_differing_app_status_is_replaced(self): + statuses = FakeResponse(200, [status_record(description="stale summary")]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [statuses], FakeResponse(201, {}))) + self.assertEqual(code, 0) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(json.loads(posts[0].data), { + "state": "success", "context": "bits-review/head", + "description": PASS_DESCRIPTION, "target_url": PASS_REVIEW_URL}) + self.assert_write_tokens(transport) + + def test_non_app_status_is_never_selected_for_comparison(self): + statuses = FakeResponse(200, [ + # Matches the decision in every compared field, but its creator + # is not the gate's publisher app. + status_record(creator=OTHER_CREATOR, status_id=900, + created_at="2026-11-12T13:00:00Z"), + # The latest app status differs, so the write still happens. + status_record(description="stale summary", status_id=50, + created_at="2026-11-12T11:00:00Z"), + ]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [statuses], FakeResponse(201, {}))) + self.assertEqual(code, 0) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(json.loads(posts[0].data)["description"], PASS_DESCRIPTION) + + def test_failed_status_read_never_suppresses_the_write(self): + code, transport = self.run_publish(publish_script( + pass_snapshot(), [FakeResponse(404)], FakeResponse(201, {}))) + self.assertEqual(code, 0) + self.assertEqual(len(self.post_calls(transport)), 1) + + def test_every_status_page_is_read_before_selection(self): + # Page 1 carries a stale app status; page 2 (through the Link + # header) a newer matching one — suppression can only come from + # reading every page. + page2 = "%s/repos/%s/commits/%s/statuses?page=2" % (API, REPO, HEAD) + first = FakeResponse(200, [status_record( + description="stale summary", status_id=50, + created_at="2026-11-12T11:00:00Z")], + link='<%s>; rel="next"' % (page2,)) + second = FakeResponse(200, [status_record( + status_id=200, created_at="2026-11-12T12:30:00Z")]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [first, second])) + self.assertEqual(code, 0) + self.assertEqual(self.post_calls(transport), []) + self.assertIn(page2, [call.full_url for call in transport.calls]) + + def test_malformed_next_link_fails_the_status_read_closed_and_still_writes(self): + # Page 1 carries a MATCHING app status plus a malformed second-page + # Link header: a successful read would have suppressed, so the + # single POST below is the fail-closed proof. + shapes = ( + '; rel="next"', + '; rel="next"', + ) + for link in shapes: + with self.subTest(link=link): + first = FakeResponse(200, [status_record()], link=link) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [first], FakeResponse(201, {}))) + self.assertEqual(code, 0) + self.assertEqual(len(self.post_calls(transport)), 1) + self.assertEqual([call.full_url for call in transport.calls].count( + statuses_read_url()), 1) + self.assertNotIn("?page=2", " ".join( + call.full_url for call in transport.calls)) + self.assert_write_tokens(transport) + + def test_malformed_app_status_is_never_selected_for_comparison(self): + # A malformed app-creator entry that matches the decision in every + # compared field is never selected for the suppression comparison: + # the write still happens. With created_at=None, guard removal + # would crash the comparison instead of suppressing. + for bad in ("created_at", "id"): + with self.subTest(bad=bad): + first = status_record(status_id=900, + created_at="2026-11-12T13:00:00Z") + if bad == "created_at": + first["created_at"] = None + else: + first["id"] = 0 + statuses = FakeResponse(200, [ + first, + status_record(description="stale summary", status_id=50, + created_at="2026-11-12T11:00:00Z"), + ]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [statuses], FakeResponse(201, {}))) + self.assertEqual(code, 0) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(json.loads(posts[0].data)["description"], + PASS_DESCRIPTION) + self.assert_write_tokens(transport) + + +class LatestSelectionTest(PublishTestCase): + """Only the app status with the greatest (created_at, id) is selected + for the comparison.""" + + def test_greatest_created_at_wins_even_when_an_older_entry_matches(self): + statuses = FakeResponse(200, [ + status_record(created_at="2026-11-12T12:00:00Z", status_id=100), + status_record(description="newer different", + created_at="2026-11-12T13:00:00Z", status_id=101), + ]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [statuses], FakeResponse(201, {}))) + self.assertEqual(code, 0) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(json.loads(posts[0].data)["description"], PASS_DESCRIPTION) + + def test_equal_created_at_is_broken_by_the_greater_id(self): + statuses = FakeResponse(200, [ + status_record(status_id=100), + status_record(status_id=101, description="same instant, higher id"), + ]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [statuses], FakeResponse(201, {}))) + self.assertEqual(code, 0) + self.assertEqual(len(self.post_calls(transport)), 1) + + +class StatusWriteTest(PublishTestCase): + """The POST: payload from the policy, the explicit sha, and the exit + code of every write branch.""" + + def test_success_is_posted_only_from_the_all_pass_branch(self): + code, transport = self.run_publish(publish_script( + pass_snapshot(), [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 0) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(posts[0].get_method(), "POST") + self.assertEqual(posts[0].full_url, status_post_url(HEAD)) + self.assertEqual(json.loads(posts[0].data), { + "state": "success", "context": "bits-review/head", + "description": PASS_DESCRIPTION, "target_url": PASS_REVIEW_URL}) + + def test_fail_and_pending_decisions_never_post_success(self): + fail = snapshot_script([member(985)], {985: [fail_node()]}) + code, transport = self.run_publish(publish_script( + fail, [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 0) + self.assertEqual( + json.loads(self.post_calls(transport)[0].data)["state"], "failure") + prs = [member(968), member(969)] + reviews = {968: [pass_node()], 969: []} + pending = snapshot_script(prs, reviews) + code, transport = self.run_publish(publish_script( + pending, [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 0) + payload = json.loads(self.post_calls(transport)[0].data) + self.assertEqual((payload["state"], payload["target_url"]), + ("pending", PR969_URL)) + + def test_error_decision_is_posted_as_error_and_exits_1(self): + code, transport = self.run_publish(publish_script( + invalid_input_snapshot(), [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 1) + payload = json.loads(self.post_calls(transport)[0].data) + self.assertEqual(payload["state"], "error") + self.assertEqual(payload["description"], INVALID_INPUT_DESCRIPTION) + + def test_member_less_error_omits_the_optional_fields_and_exits_1(self): + code, transport = self.run_publish(publish_script( + failed_pulls_script(), [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 1) + payload = json.loads(self.post_calls(transport)[0].data) + self.assertEqual(payload, {"state": "error", "context": "bits-review/head"}) + + def test_failed_post_exits_1(self): + code, transport = self.run_publish(publish_script( + pass_snapshot(), [FakeResponse(200, [])], FakeResponse(404))) + self.assertEqual(code, 1) + self.assertEqual(len(self.post_calls(transport)), 1) + + def test_missing_status_token_exits_1_without_a_post(self): + code, transport = self.run_publish(publish_script( + pass_snapshot(), [FakeResponse(200, [])]), + env=publish_env_without_status_token()) + self.assertEqual(code, 1) + self.assertEqual(self.post_calls(transport), []) + + +class WriteTokenTest(PublishTestCase): + """Every read carries GITHUB_TOKEN; the one POST carries STATUS_TOKEN + and no read ever carries it.""" + + def test_read_tokens_and_post_token_are_strictly_separated(self): + code, transport = self.run_publish(publish_script( + pass_snapshot(), [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 0) + self.assert_write_tokens(transport) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + for value in posts[0].headers.values(): + self.assertNotIn(GITHUB_TOKEN, value) + + +class ExplicitShaTest(PublishTestCase): + """The status read and the write target only the explicit --sha + argument, never any other head.""" + + def test_every_sha_bearing_url_uses_the_explicit_argument(self): + snapshot = snapshot_script( + [member(969, sha=OTHER_HEAD)], + {969: [testlib.fixture_node("pr968", commit={"oid": OTHER_HEAD})]}, + head=OTHER_HEAD) + code, transport = self.run_publish(publish_script( + snapshot, [FakeResponse(200, [])], FakeResponse(201, {})), + sha=OTHER_HEAD) + self.assertEqual(code, 0) + posts = self.post_calls(transport, sha=OTHER_HEAD) + self.assertEqual(len(posts), 1) + for call in transport.calls: + if "/statuses" in call.full_url: + self.assertIn(OTHER_HEAD, call.full_url) + self.assertNotIn(HEAD, call.full_url) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/test_bits_review_gate_workflows.py b/.github/scripts/tests/test_bits_review_gate_workflows.py new file mode 100644 index 0000000000..0bc815673a --- /dev/null +++ b/.github/scripts/tests/test_bits_review_gate_workflows.py @@ -0,0 +1,634 @@ +# Copyright 2026, Datadog, Inc. +"""Static contract tests for the Bits review gate workflow files. + +Pure stdlib text assertions over the receiver workflow: exact triggers, +empty top-level permissions with none in the job, no actions or checkout, +no expressions in ``run:`` blocks, ``timeout-minutes``, and no artifacts, +secrets or tokens. Publisher assertions extend this module: exact +triggers, trusted main checkout, SHA-pinned actions, per-job permission +sets, per-SHA concurrency without cancel, env-only inputs and the +explicit ``$SHA`` publish target. Policy assertions parse the publisher +Octo-STS file: exact issuer, subject and event claims, the protected +main ref, the workflow-ref pin and ``statuses: write`` only. Wiring +assertions extend this module as well: the ``ci.yml`` ``script-tests`` +loop runs the gate runner additively, the runner's unittest discover +pattern selects only this gate's family (an unrelated module is not +selected) while the shared helper stays importable, and AGENTS.md's +Work Tracking pointer names the workflow, the ``bits-review/head`` +context and the design doc without an activation claim. No YAML +dependency; every assertion fails when its element is removed or +mistyped. +""" + +from fnmatch import fnmatch +import os +import re +import unittest + +REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") +) +RECEIVER_WORKFLOW = os.path.join( + REPO_ROOT, ".github", "workflows", "bits-review-event.yml" +) +PUBLISHER_WORKFLOW = os.path.join( + REPO_ROOT, ".github", "workflows", "bits-review-gate.yml" +) +CHECKOUT_PIN = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1" +OCTO_STS_PIN = "DataDog/dd-octo-sts-action@c85802a58e202b7e764b5441e4378c76043c8108 # v1.0.5" +POLICY_FILE = os.path.join( + REPO_ROOT, + ".github", + "chainguard", + "self.bits-review-gate.publish-status.sts.yaml", +) +POLICY_ISSUER = "https://token.actions.githubusercontent.com" +POLICY_SUBJECT_PATTERN = ( + "repo:DataDog/java-profiler:(pull_request|ref:refs/heads/main)" +) +POLICY_EVENT_NAMES = { + "pull_request_target", + "workflow_run", + "schedule", + "workflow_dispatch", +} +POLICY_JOB_WORKFLOW_REF = ( + r"DataDog/java-profiler/\.github/workflows/bits-review-gate\.yml" + r"@refs/heads/main" +) +CI_WORKFLOW = os.path.join(REPO_ROOT, ".github", "workflows", "ci.yml") +GATE_RUNNER = ".github/scripts/tests/test_bits_review_gate.sh" +TICKET_RUNNER = ".github/scripts/tests/test_check_pr_ticket.sh" +RUNNER_SCRIPT = os.path.join(REPO_ROOT, *GATE_RUNNER.split("/")) +TESTS_DIR = os.path.dirname(RUNNER_SCRIPT) +AGENTS_FILE = os.path.join(REPO_ROOT, "AGENTS.md") +DISCOVER_PATTERN = "test_bits_review_gate_*.py" +FAMILY_MODULES = { + "test_bits_review_gate_api.py", + "test_bits_review_gate_policy.py", + "test_bits_review_gate_publish.py", + "test_bits_review_gate_workflows.py", +} +HELPER_MODULE = "bits_review_gate_testlib.py" +# Activation-claim words the AGENTS.md pointer must not use: the check is +# designed, not required, until separately approved activation lands. +POINTER_CLAIM = re.compile( + r"(?i)\b(requir(e|es|ed)|enforce[ds]?|enforcement" + r"|blocks?|blocking|must)\b" +) + + +def read_text(path): + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + + +def block_after(text, key): + """The indented block that follows a top-level ``key:`` line.""" + match = re.search(r"^%s:\n(.*?)(?=^\S|\Z)" % re.escape(key), text, re.M | re.S) + return match.group(1) if match else "" + + +def trigger_types(text, trigger): + """The exact ``types`` list of ``trigger``, or None when absent.""" + match = re.search( + r"^ %s:\n types: \[([^\]]*)\]" % re.escape(trigger), text, re.M + ) + if match is None: + return None + return [item.strip() for item in match.group(1).split(",")] + + +def run_blocks(text): + """Every ``run:`` value: the key line's remainder plus deeper-indented lines.""" + lines = text.splitlines() + blocks = [] + index = 0 + while index < len(lines): + match = re.match(r"^(\s*)run:(.*)$", lines[index]) + if match is None: + index += 1 + continue + indent, rest = match.group(1), match.group(2).strip() + block = [rest] + index += 1 + while index < len(lines): + if lines[index].strip() == "": + index += 1 + continue + deeper = re.match(r"^(\s*)\S", lines[index]) + if deeper is not None and len(deeper.group(1)) > len(indent): + block.append(lines[index].strip()) + index += 1 + else: + break + blocks.append("\n".join(block)) + return blocks + + +def sub_block(text, key): + """The child block of one 2-space-indented ``key:`` (a trigger or job). + + Ends at the next 2-space key or the end of the parent block. + """ + match = re.search( + r"^ %s:\n(.*?)(?=^ \S|\Z)" % re.escape(key), text, re.M | re.S + ) + return match.group(1) if match else "" + + +def job_block(jobs, job): + """The body of one job inside the ``jobs:`` block.""" + return sub_block(jobs, job) + + +def job_permissions(block): + """The permissions mapping of a job block, or None when absent.""" + match = re.search( + r"^ permissions:\n((?: [^\n]*\n)+)", block, re.M + ) + if match is None: + return None + entries = {} + for line in match.group(1).splitlines(): + key, _, value = line.strip().partition(": ") + entries[key] = value + return entries + + +def exact_line(block, line): + """Whether ``line`` appears as a full line, ignoring indentation.""" + return re.search(r"^[ \t]*%s$" % re.escape(line), block, re.M) is not None + + +class ReceiverWorkflowTest(unittest.TestCase): + """The no-op permissionless receiver: bits-review-event.yml.""" + + @classmethod + def setUpClass(cls): + cls.text = read_text(RECEIVER_WORKFLOW) + + def test_workflow_name(self): + self.assertIsNotNone( + re.search(r"^name: Bits review event$", self.text, re.M) + ) + + def test_triggers_are_exactly_the_two_events(self): + on_keys = set(re.findall(r"^ ([^\s:]+):", block_after(self.text, "on"), re.M)) + self.assertEqual(on_keys, {"pull_request_review", "pull_request"}) + + def test_review_trigger_types_exact(self): + self.assertEqual( + trigger_types(self.text, "pull_request_review"), + ["submitted", "edited", "dismissed"], + ) + + def test_pr_trigger_types_exact(self): + self.assertEqual(trigger_types(self.text, "pull_request"), ["edited"]) + + def test_top_level_permissions_empty(self): + self.assertEqual( + re.findall(r"^permissions:.*$", self.text, re.M), ["permissions: {}"] + ) + + def test_job_has_no_permissions(self): + self.assertEqual(re.findall(r"^[ \t]+permissions:", self.text, re.M), []) + + def test_no_actions_and_no_checkout(self): + self.assertNotIn("uses:", self.text) + self.assertNotIn("checkout", self.text.lower()) + + def test_exactly_one_static_echo_run_step(self): + blocks = run_blocks(self.text) + self.assertEqual(len(blocks), 1) + self.assertTrue(blocks[0].startswith("echo "), blocks[0]) + + def test_run_blocks_have_no_expressions(self): + for block in run_blocks(self.text): + self.assertNotIn("${{", block) + + def test_timeout_minutes_present(self): + self.assertIsNotNone( + re.search(r"^[ \t]+timeout-minutes: \d+$", self.text, re.M) + ) + + def test_exactly_one_job(self): + jobs = re.findall(r"^ ([^\s:]+):", block_after(self.text, "jobs"), re.M) + self.assertEqual(len(jobs), 1) + + def test_no_secrets_or_token_references(self): + self.assertIsNone(re.search(r"(?i)token|secret", self.text)) + + def test_no_artifacts(self): + self.assertIsNone(re.search(r"(?i)artifact", self.text)) + + +class PublisherWorkflowTest(unittest.TestCase): + """The trusted main-branch publisher: bits-review-gate.yml.""" + + @classmethod + def setUpClass(cls): + cls.text = read_text(PUBLISHER_WORKFLOW) + cls.on = block_after(cls.text, "on") + cls.jobs = block_after(cls.text, "jobs") + cls.resolve = job_block(cls.jobs, "resolve") + cls.publish = job_block(cls.jobs, "publish") + + def test_workflow_name(self): + self.assertIsNotNone( + re.search(r"^name: Bits review gate$", self.text, re.M) + ) + + def test_triggers_are_exactly_the_four_events(self): + on_keys = set(re.findall(r"^ ([^\s:]+):", self.on, re.M)) + self.assertEqual( + on_keys, + {"pull_request_target", "workflow_run", "schedule", "workflow_dispatch"}, + ) + + def test_pull_request_target_types_exact_and_unfiltered(self): + self.assertEqual( + trigger_types(self.text, "pull_request_target"), + ["opened", "reopened", "synchronize", "edited", "closed"], + ) + # A branch filter would scope the event before the resolver can. + self.assertEqual( + sub_block(self.on, "pull_request_target").split(), + ["types:", "[opened,", "reopened,", "synchronize,", "edited,", + "closed]"], + ) + + def test_workflow_run_names_receiver_exactly(self): + block = sub_block(self.on, "workflow_run") + self.assertTrue( + exact_line(block, 'workflows: ["Bits review event"]'), block + ) + self.assertTrue(exact_line(block, "types: [completed]"), block) + + def test_schedule_cron_exact(self): + self.assertTrue( + exact_line(sub_block(self.on, "schedule"), "- cron: '*/30 * * * *'") + ) + + def test_workflow_dispatch_has_no_inputs(self): + self.assertEqual(sub_block(self.on, "workflow_dispatch").strip(), "") + + def test_top_level_permissions_empty(self): + self.assertEqual( + re.findall(r"^permissions:.*$", self.text, re.M), ["permissions: {}"] + ) + + def test_exactly_two_jobs(self): + jobs = re.findall(r"^ ([^\s:]+):", self.jobs, re.M) + self.assertEqual(jobs, ["resolve", "publish"]) + + def test_resolve_guard_is_exact(self): + self.assertTrue( + exact_line(self.resolve, "if: github.ref == 'refs/heads/main'") + ) + + def test_resolve_guard_rationale_is_pinned(self): + # pull_request_target presents the base-branch ref, so the main-ref + # guard is live for PR events too; pin the rationale comment text. + self.assertIn( + "pull_request_target sets github.ref to the base branch", + self.resolve, + ) + self.assertTrue( + exact_line(self.resolve, "if: github.ref == 'refs/heads/main'") + ) + + def test_resolve_permissions_exact(self): + self.assertEqual( + job_permissions(self.resolve), + {"contents": "read", "pull-requests": "read"}, + ) + + def test_trusted_main_checkout_in_both_jobs(self): + for block in (self.resolve, self.publish): + self.assertTrue(exact_line(block, "uses: " + CHECKOUT_PIN), block) + self.assertTrue(exact_line(block, "ref: refs/heads/main"), block) + self.assertTrue( + exact_line(block, "persist-credentials: false"), block + ) + refs = re.findall(r"^ +ref: (.+)$", self.text, re.M) + self.assertEqual(refs, ["refs/heads/main", "refs/heads/main"]) + self.assertEqual( + len(re.findall(r"^ *uses: actions/checkout@", self.text, re.M)), 2 + ) + + def test_no_pr_head_checkout(self): + for banned in ( + "github.head_ref", + "github.event.pull_request.head", + "github.ref_name", + ): + self.assertNotIn(banned, self.text) + + def test_all_uses_lines_sha_pinned_with_version_comment(self): + pins = re.findall(r"^ *uses: (.+)$", self.text, re.M) + self.assertTrue(pins) + for pin in pins: + self.assertRegex(pin, r"^[^@\s]+@[0-9a-f]{40} # v[^\s]+$") + + def test_octo_sts_action_pinned_and_scoped(self): + self.assertTrue( + exact_line(self.publish, "uses: " + OCTO_STS_PIN), self.publish + ) + self.assertTrue( + exact_line(self.publish, "scope: DataDog/java-profiler"), self.publish + ) + self.assertTrue( + exact_line( + self.publish, "policy: self.bits-review-gate.publish-status" + ), + self.publish, + ) + + def test_resolve_inputs_arrive_through_env_only(self): + env_lines = ( + "GITHUB_TOKEN: ${{ github.token }}", + "GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}", + "GITHUB_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}", + "GITHUB_HEAD_REPOSITORY: " + "${{ github.event.workflow_run.head_repository.full_name }}", + "GITHUB_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}", + "GITHUB_PULL_REQUEST_NUMBERS: " + "${{ toJSON(github.event.workflow_run.pull_requests[*].number) }}", + "GITHUB_PR_ACTION: ${{ github.event.action }}", + "GITHUB_CHANGES_BASE_REF: ${{ github.event.changes.base.ref.from }}", + "GITHUB_CHANGES_BASE_SHA: ${{ github.event.changes.base.sha.from }}", + ) + for line in env_lines: + self.assertTrue(exact_line(self.resolve, line), line) + + def test_resolve_run_command_exact(self): + self.assertEqual( + run_blocks(self.resolve), + ["python3 .github/scripts/bits_review_gate.py resolve"], + ) + + def test_resolve_step_output_exposed_as_job_output(self): + self.assertTrue(exact_line(self.resolve, "id: resolve"), self.resolve) + self.assertTrue( + exact_line( + self.resolve, "shas: ${{ steps.resolve.outputs.shas }}" + ), + self.resolve, + ) + + def test_publish_needs_and_guard(self): + self.assertTrue(exact_line(self.publish, "needs: resolve"), self.publish) + self.assertTrue( + exact_line( + self.publish, + "if: github.ref == 'refs/heads/main' " + "&& needs.resolve.outputs.shas != '[]'", + ), + self.publish, + ) + + def test_publish_permissions_exact(self): + self.assertEqual( + job_permissions(self.publish), + { + "contents": "read", + "pull-requests": "read", + "statuses": "read", + "id-token": "write", + }, + ) + + def test_matrix_sha_from_resolver_output(self): + self.assertTrue( + exact_line( + self.publish, + "sha: ${{ fromJSON(needs.resolve.outputs.shas) }}", + ), + self.publish, + ) + self.assertTrue( + exact_line(self.publish, "fail-fast: false"), self.publish + ) + + def test_per_sha_concurrency_without_cancel(self): + self.assertTrue( + exact_line( + self.publish, "group: bits-review-gate-${{ matrix.sha }}" + ), + self.publish, + ) + self.assertTrue( + exact_line(self.publish, "cancel-in-progress: false"), self.publish + ) + + def test_no_environment_key(self): + self.assertIsNone(re.search(r"^[ \t]+environment:", self.text, re.M)) + + def test_run_blocks_have_no_expressions(self): + for block in run_blocks(self.text): + self.assertNotIn("${{", block) + + def test_publish_targets_explicit_matrix_sha(self): + self.assertEqual( + run_blocks(self.publish), + ['python3 .github/scripts/bits_review_gate.py publish --sha "$SHA"'], + ) + for line in ( + "SHA: ${{ matrix.sha }}", + "GITHUB_TOKEN: ${{ github.token }}", + "STATUS_TOKEN: ${{ steps.octo-sts.outputs.token }}", + ): + self.assertTrue(exact_line(self.publish, line), line) + self.assertNotIn("GITHUB_SHA", self.text) + self.assertNotIn("github.sha", self.text) + + def test_no_artifacts(self): + self.assertNotIn("artifact", self.text.lower()) + + +class PublisherOctoStsPolicyTest(unittest.TestCase): + """The status-write-only publisher policy: the Octo-STS claim pins.""" + + @classmethod + def setUpClass(cls): + cls.text = read_text(POLICY_FILE) + + def top_level_value(self, key): + """The single ``key:`` value at the top level, or None when absent.""" + values = re.findall(r"^%s: (.+)$" % re.escape(key), self.text, re.M) + return values[0] if values else None + + def claim_value(self, claim): + """The value of one ``claim_pattern`` entry, or None when absent.""" + values = re.findall(r"^ %s: (.+)$" % re.escape(claim), self.text, re.M) + return values[0] if values else None + + def test_issuer_is_github_actions_oidc(self): + self.assertEqual(self.top_level_value("issuer"), POLICY_ISSUER) + + def test_subject_pattern_covers_exactly_both_forms(self): + self.assertEqual( + self.top_level_value("subject_pattern"), POLICY_SUBJECT_PATTERN + ) + forms = re.findall( + r"^subject_pattern: repo:DataDog/java-profiler:\(([^)]*)\)$", + self.text, + re.M, + ) + self.assertEqual(len(forms), 1) + self.assertEqual(forms[0].split("|"), ["pull_request", "ref:refs/heads/main"]) + + def test_event_names_are_exactly_the_four(self): + events = re.findall( + r"^ event_name: \(([^)]*)\)$", self.text, re.M + ) + self.assertEqual(len(events), 1) + self.assertEqual(set(events[0].split("|")), POLICY_EVENT_NAMES) + + def test_ref_is_protected_main(self): + self.assertEqual(self.claim_value("ref"), "refs/heads/main") + self.assertEqual(self.claim_value("ref_protected"), '"true"') + + def test_job_workflow_ref_pinned_to_gate_on_main(self): + self.assertEqual( + self.claim_value("job_workflow_ref"), POLICY_JOB_WORKFLOW_REF + ) + + def test_permissions_are_statuses_write_only(self): + block = re.search( + r"^permissions:\n((?: [^\n]+\n)+)", self.text, re.M + ) + self.assertIsNotNone(block, self.text) + entries = {} + for line in block.group(1).splitlines(): + key, _, value = line.strip().partition(": ") + entries[key] = value + self.assertEqual(entries, {"statuses": "write"}) + + +class CiScriptTestsWiringTest(unittest.TestCase): + """The additive script-tests loop entry for the gate runner in ci.yml.""" + + @classmethod + def setUpClass(cls): + cls.jobs = block_after(read_text(CI_WORKFLOW), "jobs") + cls.job = sub_block(cls.jobs, "script-tests") + + def loop_entries(self): + """The whitespace-separated paths of the job's ``for t in`` loop.""" + match = re.search(r"^ +for t in (.+); do$", self.job, re.M) + self.assertIsNotNone(match, self.job) + return match.group(1).split() + + def test_script_tests_job_exists(self): + self.assertNotEqual(self.job, "") + + def test_loop_runs_gate_runner(self): + self.assertIn(GATE_RUNNER, self.loop_entries()) + + def test_loop_keeps_existing_entries(self): + # Additive only: the PR-ticket runner and the utils glob stay. + entries = self.loop_entries() + self.assertIn("utils/tests/*.sh", entries) + self.assertIn(TICKET_RUNNER, entries) + + +class RunnerDiscoveryTest(unittest.TestCase): + """AC-12: the gate runner discovers only this gate's test family.""" + + @classmethod + def setUpClass(cls): + cls.text = read_text(RUNNER_SCRIPT) + + def discover_pattern(self): + """The single ``-p`` pattern of the runner's discover command.""" + patterns = re.findall(r"-p '([^']+)'", self.text) + self.assertEqual(len(patterns), 1, patterns) + return patterns[0] + + def test_pattern_is_exactly_the_gate_family(self): + self.assertEqual(self.discover_pattern(), DISCOVER_PATTERN) + + def test_pattern_selects_only_the_family_modules_present(self): + pattern = self.discover_pattern() + matched = { + name for name in os.listdir(TESTS_DIR) if fnmatch(name, pattern) + } + self.assertEqual(matched, FAMILY_MODULES) + + def test_pattern_selects_no_unrelated_module(self): + pattern = self.discover_pattern() + for name in ( + "test_other.py", + "test_check_pr_ticket.py", + "test_quarantine.py", + HELPER_MODULE, + "run_release_cli.py", + GATE_RUNNER.split("/")[-1], + ): + self.assertFalse(fnmatch(name, pattern), name) + + def test_helper_import_stays_legal(self): + # The helper is never discovered as a suite, but family modules + # import it, so discovery of the family depends on it existing. + self.assertTrue( + os.path.isfile(os.path.join(TESTS_DIR, HELPER_MODULE)), HELPER_MODULE + ) + importers = [ + name + for name in sorted(FAMILY_MODULES) + if "bits_review_gate_testlib" in read_text( + os.path.join(TESTS_DIR, name) + ) + ] + self.assertTrue(importers, "no family module imports the helper") + + +class AgentsWorkTrackingPointerTest(unittest.TestCase): + """The concise Work Tracking pointer to the bits review gate design.""" + + @classmethod + def setUpClass(cls): + cls.text = read_text(AGENTS_FILE) + match = re.search( + r"^## Work Tracking\n(.*?)(?=^## |\Z)", cls.text, re.M | re.S + ) + cls.section = match.group(1) if match else "" + + def pointer_paragraph(self): + """The wrapped paragraph of the pointer, joined onto one line.""" + lines = self.section.splitlines() + starts = [i for i, line in enumerate(lines) if "bits-review-gate" in line] + self.assertEqual(len(starts), 1, starts) + paragraph = [lines[starts[0]]] + for line in lines[starts[0] + 1 :]: + if not line.strip(): + break + paragraph.append(line) + return " ".join(paragraph) + + def test_pointer_follows_the_pr_ticket_sentence(self): + ticket = self.section.find(".github/workflows/pr-ticket.yml") + self.assertNotEqual(ticket, -1) + self.assertGreater(self.section.find("bits-review-gate"), ticket) + + def test_pointer_names_workflow_context_and_design_doc(self): + paragraph = self.pointer_paragraph() + for token in ( + "bits-review-gate", + "bits-review/head", + "doc/BitsReviewMergeGateDesign.md", + ): + self.assertIn(token, paragraph) + + def test_pointer_makes_no_activation_claim(self): + paragraph = self.pointer_paragraph() + self.assertIsNone(POINTER_CLAIM.search(paragraph), paragraph) + + def test_pointer_is_the_only_gate_mention_in_agents_md(self): + self.assertEqual(self.text.count("bits-review-gate"), 1) + + +if __name__ == "__main__": + unittest.main() From a3c8e5fd7a3f98a2b44edff1847cbf70d1246f0b Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 13:57:42 +0100 Subject: [PATCH 03/11] Document the Bits review merge gate and wire CI to it --- .github/workflows/ci.yml | 2 +- AGENTS.md | 5 + doc/BitsReviewMergeGateDesign.md | 447 +++++++++++++++++++++++++++++++ doc/README.md | 1 + 4 files changed, 454 insertions(+), 1 deletion(-) create mode 100644 doc/BitsReviewMergeGateDesign.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f9be9e649..8969496cd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: sudo apt-get update sudo apt-get install -y binutils g++ jq status=0 - for t in utils/tests/*.sh .github/scripts/tests/test_check_pr_ticket.sh; do + for t in utils/tests/*.sh .github/scripts/tests/test_check_pr_ticket.sh .github/scripts/tests/test_bits_review_gate.sh; do echo "::group::$t" bash "$t" || { echo "::error::$t failed"; status=1; } echo "::endgroup::" diff --git a/AGENTS.md b/AGENTS.md index 611684c8fc..89481b3a23 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,6 +173,7 @@ before working on a subsystem. | Per-sampler perf counters (`-PenableSamplerPerf`) | [doc/reference/SamplerPerfCounters.md](doc/reference/SamplerPerfCounters.md) | | Test flakiness analysis | [doc/reference/TestFlakinessAnalysis.md](doc/reference/TestFlakinessAnalysis.md) | | Demo leak app (`demo-java-leak`: self-driven HTTP server with a strong-ref leak cache and post-GC watermark flush; no profiler dependency) | [doc/architecture/DemoJavaLeakApp.md](doc/architecture/DemoJavaLeakApp.md) | +| Bits review merge gate (CI status publisher/policy: trusted-bot allowlist, PASS bound to the exact head SHA and shared as a conjunction across main PRs — never a transferable approval, no fallback or override) | [doc/BitsReviewMergeGateDesign.md](doc/BitsReviewMergeGateDesign.md) | Key source locations: Java API `ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java`; C++ engine `ddprof-lib/src/main/cpp/` (implementation-specific code under `hotspot/`, `j9/`, @@ -499,6 +500,10 @@ it must be disclosed per the rule at the top of this file. These rules always ap a GitHub issue `#`. Enforced in CI: the `PR ticket reference` workflow (`.github/workflows/pr-ticket.yml`) fails the PR otherwise; bot PRs are exempt. +The `bits-review-gate` workflow publishes the `bits-review/head` commit status +for PR heads targeting `main`; see [doc/BitsReviewMergeGateDesign.md](doc/BitsReviewMergeGateDesign.md) +for the design and its current activation status. + Isolation, parallel sessions and the rest of the tracking contract are in [doc/WorkTracking.md](doc/WorkTracking.md) and enforced by review discipline. Existing production-change guidelines and review rules still apply in full. diff --git a/doc/BitsReviewMergeGateDesign.md b/doc/BitsReviewMergeGateDesign.md new file mode 100644 index 0000000000..4337372d2f --- /dev/null +++ b/doc/BitsReviewMergeGateDesign.md @@ -0,0 +1,447 @@ + + +# Bits Review Merge Gate — Specification and Design + +**Status:** The gate is implemented in this change set and its hermetic local +tests pass. It is not activated: `bits-review/head` is not a required check +on `main`, no protection setting or Octo-STS installation state was changed, and +no enforcement claim applies. Activation needs separate approval. + +## Goal and scope + +Require a completed Bits Code Review PASS for the current head of each pull +request targeting `main`. Apply the rule to human, bot, dependency, documentation, +and fork pull requests. Do not add path-based or author-based exemptions. + +The gate is an additional merge requirement. It does not replace CI, signing, +human reviews, or conversation resolution. Preserve existing admin bypass rights. +Do not add a label, comment, or maintainer override. + +## Evidence and terminology + +A **result** is a published Bits summary review, not an inline finding or an +ordinary approval. The two supplied examples are GitHub `COMMENTED` reviews: + +| Example | Review commit | Metadata verdict | +| --- | --- | --- | +| [Review on PR 985](https://github.com/DataDog/java-profiler/pull/985#pullrequestreview-5872953227) | `1dcc3fa57582f2b595403f2b6a54e4464f352223` | FAIL | +| [Review on PR 968](https://github.com/DataDog/java-profiler/pull/968#pullrequestreview-5872101609) | `1115eb583f475b62544d634bb12bf750ae913e04` | PASS | + +Their bodies contain `autotest-meta` metadata with a `verdict` field. The supplied +REST records use `commit_id`; the evaluator reads GraphQL `commit.oid`. +Use the API commit field, not badge text, a shortened hash, or a body link. + +The [Bits onboarding documentation](https://datadoghq.atlassian.net/wiki/spaces/Autotest/pages/7297566123) +describes these comments as non-blocking. Requiring GitHub approvals does not +turn the metadata verdict into a merge requirement. + +## Policy contract + +Bind evaluation to the repository, pull request, and exact current head SHA. +Only a result from that pull request and SHA can satisfy the policy. + +| Condition | Required outcome | +| --- | --- | +| Latest candidate is a trusted, valid, never-edited PASS for the current head | Pass the Bits requirement | +| Latest candidate is a trusted FAIL for the current head | Block | +| No completed result exists for the current head | Block | +| PASS exists only for another SHA or another pull request | Block | +| Latest result is dismissed or its verdict metadata is invalid | Block; do not fall back to an older PASS | +| Latest bot candidate has an actual body edit | Block, even if the original bot made the edit; require a fresh summary review | +| Latest Bits-marked result is from an unlisted bot | Block as untrusted origin; it cannot pass | +| A newer valid result replaces FAIL with PASS on the same SHA | Pass | +| A newer valid result replaces PASS with FAIL on the same SHA | Block after reevaluation | +| Another review starts after PASS on the same SHA | Keep the completed PASS until a new result or invalidation arrives | +| Push or rebase changes the head SHA | Require a result for the new SHA | +| Evaluation cannot establish a valid current PASS | Do not publish success | + +“Latest” means submission order among candidate Bot reviews for the same PR and +SHA, including untrusted origins. Compare submission times as UTC instants, with +review ID as a tie-breaker. An edit does not make an older review newer. +Webhook arrival order is not authority. + +Select the latest result before checking its dismissal state and verdict validity. +Filtering invalid or dismissed results first would allow an older PASS to reappear. +An unrelated review must not satisfy the requirement. Two conservative exceptions +can block: a latest Bits-marked review from an unlisted bot, and a latest edited +review from any bot whose markers are absent and whose origin cannot be established. +Copied markers from a User account are ignored. + +A dismissed FAIL is not a PASS. Resolving findings, replying to them, or receiving +human approval does not clear FAIL. A new valid Bits PASS or an existing authorized +admin bypass is needed. + +## Architecture + +The gate uses a project-owned evaluator and publisher. It publishes under the +dedicated, stable status context `bits-review/head`. The context is not a +required check until activation. + +```text +PR/review events + recovery sweep + manual retry + | + trusted per-SHA reevaluation + | + all open main PRs sharing that head + | + separate result check per PR + | + success only if every PR passes + | + required status check on main +``` + +The shared-SHA result is a conjunction, not a transferable approval. An unreviewed +duplicate PR blocks the reviewed PR until the duplicate gets PASS or closes, or +an existing authorized admin bypass is used. This block-only denial risk is accepted. + +### Trusted result recognition + +The approved allowlist contains the numeric identities below. Each must also have +GitHub actor type `Bot`. The allowlist belongs in trusted main-branch code. + +| Login | GitHub user ID | +| --- | --- | +| `datadog-official[bot]` | `214633350` | +| `datadog-datadog-prod-us1[bot]` | `88084959` | +| `datadog-datadog-prod-us1-2[bot]` | `261164178` | +| `datadog-datadog-us1-prod[bot]` | `298424163` | +| `datadog-prod-us1-3[bot]` | `266080212` | +| `datadog-prod-us1-4[bot]` | `266080418` | +| `datadog-prod-us1-5[bot]` | `266081015` | +| `datadog-prod-us1-6[bot]` | `266788760` | + +PASS requires trusted origin and validated Bits summary metadata, including the +`datadog-code-review` reviewer ID `bad90057-5c12-4101-ad47-d91c9dff0f90` and +`datadog-autotest:v1:prod` marker. A badge alone never establishes a result. + +A latest Bits-marked review from an unlisted bot blocks as untrusted origin, even +if its text says PASS. This prevents an unlisted shard's FAIL from being ignored. +A bot capable of copying the markers can cause a false block, not a false PASS. + +A submitted Bot review on the exact head is a candidate when its body contains +`autotest-meta`, `datadog-code-review`, or `datadog-autotest`, or when it was edited. +An edited markerless review is included even from an unlisted bot. An unedited, +unmarked Bot review is ignored. Pending reviews and null commit references are +out of scope; known User-authored reviews are ignored. + +Every latest edited bot candidate blocks, including a valid-looking summary edited +by its original bot. Require a fresh trusted summary review to clear that block. +This avoids trusting the current body solely from its original author identity. + +Over-blocking unrelated edited bot reviews is accepted. This also closes the case +where an unlisted shard erases its markers and would otherwise expose an older +trusted PASS. A newer trusted valid Bits PASS can clear the conservative block. + +Determine an actual edit only from GraphQL `lastEditedAt != null`. Do not treat a +creation revision or edit-history count as edit proof. Do not query or trust editor +history to permit an edited PASS. A [bot-authored review with a human editor](https://github.com/DataDog/java-profiler/pull/615#pullrequestreview-4565721745) +provides a concrete reason for this restriction. + +GitHub's signal for an app-authored body edit must be proved in a live test before +activation. If it is absent, return the provenance design for a decision instead +of weakening the no-fallback rule. + +### Evaluation and publication + +The Python evaluator (`.github/scripts/bits_review_gate.py` and +`.github/scripts/bits_review_gate_policy.py`) uses only the standard library. +Its policy functions consume normalized records; an injectable API transport +supports hermetic tests. The CLI has `resolve` and `publish --sha H` subcommands. + +Read every page of the REST open-PR list and the GraphQL review connection. Preserve +review ID, actor type, numeric actor ID, login, state, body, commit OID, submission +time, and raw `lastEditedAt`. Do not fetch edit-history records. + +GraphQL errors, including partial-data responses, missing required response +structure, and transport failures produce `api-error`. Malformed required PR data +or possible-candidate data produces `invalid-input`. A marked or edited review +with an unclassifiable author or ordering field must not expose an older PASS. +Known User-authored reviews remain ignored. Network and server errors get at most +three attempts; client errors are not retried. + +After validation, use this outcome order: + +| Condition | Reason | Commit status | +| --- | --- | --- | +| Snapshot API failure or invalid required data | `api-error` or `invalid-input` | `error` | +| No candidate | `missing` | `pending` | +| Latest candidate dismissed | `dismissed` | `failure` | +| Latest candidate from an unlisted Bot | `untrusted-origin` | `failure` | +| Latest allowlisted candidate edited | `edited-summary` | `failure` | +| Latest allowlisted candidate has invalid markers | `malformed:` | `failure` | +| Valid metadata says FAIL | `fail` | `failure` | +| Valid metadata says PASS | `pass` | `success` | + +Valid metadata has one `autotest-meta` comment with exactly one case-sensitive +`verdict=PASS` or `verdict=FAIL` token, one matching reviewer-ID comment, and the +production environment marker with no conflicting environment marker. Malformed +sub-reasons distinguish missing metadata, unknown or conflicting verdicts, +reviewer-ID mismatch, and an invalid environment marker. Ignore badge text and +delivery markers when deciding the verdict. + +Conjoin the separate outcomes for all open `main` PRs whose head is H, including +drafts. Severity is `error`, then `failure`, then `pending`, then `success`. +The lowest PR number governs equal-severity outcomes. Link the governing review, +or its PR when no review governs. Build URLs from API numbers, not review text. +Limit the status description to 140 characters and include its reason. + +Serialize writers with concurrency group `bits-review-gate-` and +`cancel-in-progress: false`. The writer reads snapshot S1 and computes its decision, +then reads S2. Compare SHA-256 fingerprints of canonical JSON covering PR +membership/head/base/state, every normalized review field, body digests, and error +flags with their locations. Retain raw edit times and identity/order inputs. + +If fingerprints differ, discard the earlier decision and continue from the new +snapshot. After three unstable comparisons, exit without writing. If the open-PR +set for H was read successfully and is empty, write nothing. Never infer H from +the workflow's `GITHUB_SHA`. + +Before writing, read all status pages and select the newest `bits-review/head` +entry created by `dd-octo-sts[bot]`, ordered by creation time and ID. Suppress the +write only when its state, description, and target URL match the desired status. +A failed status read must not suppress a safety update. An error decision fails +the run even when an unchanged-status match suppresses its POST. Failed writes +also fail the run. + +Only the all-PASS branch may publish success. Failure to start or write, or unstable +snapshots with no write, can leave an older status visible. These paths do not +establish that GitHub is blocking merges. + +### Events, permissions, and environments + +Cover pull request creation, reopening, head synchronization, and retargeting to +`main`. Cover result submission, editing, and dismissal. A retry must reevaluate +live state; it must not replay a saved success. + +The writer must run trusted base-branch code with only the permissions it needs. +Never execute PR code, interpolate review bodies into shell commands, or execute +artifacts supplied by an untrusted workflow. + +Support GitHub-hosted Linux runners, same-repository PRs, fork PRs, and dependency +bot PRs. Fork and bot permission restrictions must be tested. A read-only +`pull_request_review` job alone is not a complete publishing design. + +Use a hermetic evaluator test suite that works in a clean checkout without GitHub +credentials. Agent harness choice must not affect the merge decision; local hooks +and agent instructions are not enforcement. + +The no-op receiver listens to `pull_request_review` submitted, edited, and dismissed +events. It also listens to `pull_request` edited events as a retarget-away hint. +It has empty permissions, no checkout, no artifacts, and no token-bearing step. + +The publisher listens to receiver `workflow_run` completion and unfiltered +`pull_request_target` opened, reopened, synchronize, edited, and closed events +(the resolver guard drops title/body-only `edited` runs; see resolver behavior +below). Both publisher jobs require `github.ref == 'refs/heads/main'`. They check out +`refs/heads/main` with `persist-credentials: false`. Pin third-party actions to +full commit SHAs. Pass event selectors through environment variables, never +interpolated into shell commands. + +The resolver has content-read and pull-request-read permissions. Per-SHA publish +jobs additionally have status-read and OIDC token-write permissions. The read-only +GitHub token never publishes the authoritative status. Do not set a job environment +that changes the OIDC subject. + +Resolver behavior is fixed: + +- PR events resolve the PR's live head by number, including close and retarget events. + A `pull_request_target` `edited` event with no base change (title/body-only edit) + resolves to no SHAs: the resolver guard fails open on the env-only selectors + `GITHUB_PR_ACTION`, `GITHUB_CHANGES_BASE_REF`, and `GITHUB_CHANGES_BASE_SHA`, so + only retarget edits resolve. +- Receiver completion resolves its run head plus live open-main heads matching PR + numbers or head repository/branch. If none match, scan all open-main heads. +- Schedule and input-free manual dispatch resolve all open-main heads. +- Validate numbers and full SHAs, deduplicate, and fail rather than truncate if the + result exceeds 256 distinct SHAs. Emit a JSON array through `GITHUB_OUTPUT`. + +A 30-minute scheduled sweep provides recovery, not a maximum-refresh or availability +promise. Fork associations, token claims, and permission behavior need a test drive. + +### Repository enforcement + +The selected activation proposal adds the dedicated status to the existing classic +branch protection on `main`. Do not add a second ruleset. Preserve unrelated +protections and existing admin bypass. A workflow without a required status does +not enforce the merge policy. + +Use the existing `dd-octo-sts` app as publisher, with a repository policy that +grants only status-write authority. Bind issuance to the trusted gate workflow on +`refs/heads/main`, the protected-ref claim, and its allowed event types. Use a +read-only `GITHUB_TOKEN` for reads. Do not use it to publish the authoritative status. + +App-source binding does not isolate the gate from internal services that also hold +status-write authority as `dd-octo-sts`. Those services are accepted as trusted +infrastructure. Candidate-controlled workflows must not obtain the publisher token. +The installation permission and exact policy claims must be tested before activation. +The repository policy file `.github/chainguard/self.bits-review-gate.publish-status.sts.yaml` +carries exactly these claims: + +```yaml +issuer: https://token.actions.githubusercontent.com +subject_pattern: repo:DataDog/java-profiler:(pull_request|ref:refs/heads/main) +claim_pattern: + event_name: (pull_request_target|workflow_run|schedule|workflow_dispatch) + ref: refs/heads/main + ref_protected: "true" + job_workflow_ref: DataDog/java-profiler/\.github/workflows/bits-review-gate\.yml@refs/heads/main +permissions: + statuses: write +``` + +Octo-STS matches every pattern against the entire claim or subject value: it +compiles `subject_pattern` and each `claim_pattern` entry as `^(?:)$` +(full-value matching; anchored compilation since v0.10.0, fixing +GHSA-mwqh-2vg8-rhj3). The unanchored alternations above are therefore whole-value +matches: event_name `pull_request` or ref `refs/heads/main-x` is rejected. Do not +add manual `^...$` anchors: around a top-level `|` they bind as `(^A)|(B$)`, the +partial-match leak the app-level wrapping exists to prevent. + +When separately authorized, propose required context `bits-review/head` with expected +publisher app ID `1157446`. Read the live protection and rules into a fresh baseline +immediately before an authorized change. The change is additive: preserve every +existing required context and check, the existing `strict` value, `enforce_admins`, +and every unrelated field. Initialize the gate's context with `strict: false` only +when the fresh baseline has no required checks; never replace the checks list +unconditionally or disable an existing strict requirement. If the fresh baseline +differs from these assumptions, stop and revise the proposal before any authorized +write. Read back and compare the settings against the fresh baseline after the change. + +GitHub status storage is commit-scoped. The conjunction across shared-head PRs is +the approved isolation policy. Prove that one PR's review cannot satisfy another +PR and that closing a blocking duplicate causes reevaluation. + +## Consistency and availability limits + +The accepted consistency model is asynchronous. A later FAIL or invalidation can +arrive before GitHub receives the blocking status. Normal processing delay is +accepted; this design does not claim an atomic review-and-merge operation. + +A delivery outage or a writer that cannot start or publish can leave an earlier +success visible on the same SHA. A newly opened PR that reuses a green SHA can +also see that status before shared-head reevaluation. This is not an atomic +PR-admission gate. Conservative reads do not remove these external windows. + +A running review on a SHA with PASS does not revoke that PASS by itself. This design +does not require Bits review-start signals, automatically trigger reviews, or change +Bits rollout settings. A PR that never gets a result remains blocked by policy. + +Do not add a numerical freshness or delivery-latency promise without measurement +and approval. Stronger guarantees need a separate lifecycle or merge-path design. + +## Alternatives not selected + +| Alternative | Reason | +| --- | --- | +| Require ordinary GitHub approvals | Does not enforce the Bits metadata verdict | +| Require conversation resolution | Does not establish a current-head PASS | +| Match a badge or any PASS text | Does not establish trusted origin, latest result, or reviewed SHA | +| Add only local hooks or agent instructions | Does not enforce ordinary GitHub merges | +| Use the documented MergeGate ARP rule | Not the specified Bits verdict contract; documented to fail open on ARP errors | +| Add native Bits or Devflow support first | Not the selected project-owned route | + +The [MergeGate documentation](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/4948100574/MergeGate) +is the source for the documented ARP behavior. No claim is made that every possible +platform configuration has been ruled out. + +## Acceptance and verification + +The local cases are implemented as permanent hermetic tests and pass; they make no +claim about live GitHub behavior. Test-drive and production-check cases are one-time +evidence tied to the exact candidate, configuration, and observed head SHA; they +remain open until activation is separately authorized. + +| ID | Environment | Required evidence | +| --- | --- | --- | +| AC1 | local | Fixtures from the supplied reviews produce FAIL and PASS for their respective SHAs; shortened or wrong SHAs do not match | +| AC2 | local | Missing, stale, wrong-PR, and wrong-author results cannot pass; all approved account identities are covered; a latest Bits-marked unknown-bot result blocks | +| AC3 | local | PASS→FAIL and FAIL→PASS follow submission order, including timestamp ties, out-of-order events, and an edit to an older review | +| AC4 | local | Dismissed, edited, or malformed latest candidates block without fallback; valid-looking original-author edits cannot pass; erased markers and unrelated edited bots cannot expose an older PASS | +| AC5 | local | Pagination, unknown or conflicting verdicts, API errors, and unrelated reviews take the specified branches | +| AC6 | local | Controlled interleavings prove head changes and stale runs cannot publish success for the wrong candidate or overwrite a newer decision | +| AC7 | test-drive | Same-repository, fork, and dependency bot PRs receive status updates without running PR-controlled code in the privileged writer | +| AC8 | test-drive | A push invalidates an earlier PASS; submission, edit, dismissal, and retry update the head status; an actual bot body edit exposes the edit signal used by the evaluator | +| AC9 | test-drive | A branch workflow cannot obtain publisher authority; each shared-head PR needs its own PASS; a missing-review duplicate blocks and closing it reevaluates the SHA | +| AC10 | production-check | Over a controlled test PR lifecycle, an ordinary non-bypass actor cannot merge without PASS and becomes eligible with PASS, subject to other checks | +| AC11 | production-check | Read-back confirms the required publisher, main-only scope, unchanged unrelated protections, and preserved admin bypass | +| AC12 | test-drive | Token exchange is denied for event_name `pull_request` and for ref `refs/heads/main-x` (no publisher token minted), proving full-value claim matching before activation | + +Run the repository's adjacent script tests and CI safety checks for implementation +changes. Run `:ddprof-lib:qualityChecks` through the documented build wrapper for +documentation changes. Live checks must not merge a test PR or change protection +settings without explicit authorization. + +## Rollout, recovery, and review + +The design and policy contract are approved; the implementation is complete in +this change set. The evaluator, receiver, publisher, and repository policy file +are in place, with hermetic local tests passing. No test PR has been driven and +no live path is claimed. + +1. On authorized test PRs, validate the real event and permission paths while the + status is not required. No enforcement claim applies yet. +2. Obtain separate approval to require the status on `main`. Read back the settings + and run the production checks with a non-bypass identity. + +Security and repository-access-control review is required for the privileged writer, +publisher authority, and enforcement settings. No profiler runtime, public API, +JFR schema, or signal-handler behavior changes are included. + +For an isolated false positive or outage, use only the existing authorized admin +bypass. A rollback of the required-check setting is a separate administrative +change needing explicit approval; record that enforcement is then disabled. + +The evaluator reads review metadata only. It adds no profiler runtime overhead and +runs no LLM inference. GitHub API traffic scales with the review pages inspected +per reevaluation; avoid fetching source trees or rerunning the build matrix. + +## Design synchronization and significant deviations + +This document is the design source of truth through merge. Update it with each +implementation change that alters the described behavior, authority boundary, +interface, failure mode, or verification method. Reconcile it against the full +candidate before requesting review and before merge. + +Record significant necessary deviations here with their reason, policy impact, +approval state, and required verification. A proposed deviation is not permission +to implement it. Keep the policy contract and acceptance criteria consistent with +approved decisions; do not weaken them to make an implementation pass. + +The following material refinements are approved and reflected in the contract above: + +| Decision | Reason and accepted impact | Required verification | +| --- | --- | --- | +| Eight-account allowlist; unknown marked bots block | A two-account allowlist can ignore a newer shard's FAIL; copied bot markers may cause false blocks | Identity, ordering, unknown-origin, and marker-copy cases | +| Conservative edited-bot candidates, including unknown origins | Erased metadata must not uncover an older PASS; unrelated edited bot reviews may block | Actual-edit detection, unknown-origin erasure, and live bot-edit signal | +| No edited summary can pass | Original author identity does not authenticate later text; even original-bot edits require a fresh review | Valid-looking edited PASS, original-author edit, and fresh-review recovery cases | +| Main-workflow-bound Octo-STS publisher | Branch workflows can share the Actions identity; existing internal Octo-STS status writers remain trusted | Token-claim restrictions and attempted branch-workflow spoofing | +| Shared-SHA conjunction | Statuses cannot express independent PR approval; a duplicate PR may block other PRs on the same head | Separate results per PR, duplicate closure and retargeting | + +The implementation also includes event-driven updates, a 30-minute recovery sweep, +per-SHA serialization, and unchanged-status suppression. Their decision logic is +covered by the local hermetic tests; live delivery of the updates is test-drive +evidence (AC7–AC9). + +The completed implementation records no significant deviation from this design: +every described behavior, authority boundary, and verification method is implemented +as specified, with no policy weakening. + +## Implementation and activation boundaries + +The technical choices above are the implemented baseline. The evaluator and its +tests use only the standard library. The tests are split by policy, API, publication, +and workflow contracts, plus a hermetic runner; each file stays reviewable. + +Activation remains separate. It requires live proof of app-authored edit detection, +API identity mapping, fork and dependency-bot delivery, token-claim restrictions, +expected publisher binding, and non-bypass enforcement. If a technical constraint +requires changing the policy, stop and obtain approval before changing the design +or implementing that deviation. + +## GitHub references + +- [Workflow events and fork behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows) +- [Secure use of pull_request_target](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target) +- [Required status checks and successful conclusions](https://docs.github.com/en/pull-requests/how-tos/merge-and-close-pull-requests/troubleshooting-required-status-checks) +- [Protected branches and expected status publishers](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches) diff --git a/doc/README.md b/doc/README.md index 3eb77891ce..ca5c73132d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -51,6 +51,7 @@ All documentation files use **PascalCase** naming (e.g., `BuildSystemGuide.md`). ### Design +- [BitsReviewMergeGateDesign](BitsReviewMergeGateDesign.md) - Bits PASS merge gate: policy, implementation, trusted status publisher, and enforcement proposal pending separate approval - [ReferenceChainsCollectionSummary](ReferenceChainsCollectionSummary.md) - Short reference-chain overview and limits - [ReferenceChainsDesign](ReferenceChainsDesign.md) - Reference-chain design decision and trade-offs - [WorkTracking](WorkTracking.md) - Work-tracking phases: Intent, Design, Build and Test, Deploy From 26a385f3d2dd5cffcf93c60c9e33fc3402334a0e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 15:39:06 +0100 Subject: [PATCH 04/11] Restore two-snapshot revalidation in the bits review gate revalidated_decision again reads S(i), computes D(i), re-reads and fingerprint-compares S(i+1); a mismatch discards D(i) and restarts from the new snapshot, three mismatches return unstable and the publish CLI exits 1 without writing. Drops an unused import from the publish tests. --- .github/scripts/bits_review_gate.py | 52 +++-- .../tests/test_bits_review_gate_publish.py | 211 ++++++++++++++++-- 2 files changed, 225 insertions(+), 38 deletions(-) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index fc12a70938..2b8ba905b7 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -13,9 +13,11 @@ the policy's record shape and ``read_sha_snapshot`` wires the policy validators into the fetch path, recording api-error and invalid-input flags instead of skipping or truncating malformed input. -``revalidated_decision`` composes the decision on that seam: it reads one -snapshot and computes the decision with the policy — return values only: -no status is read or written there. The ``publish --sha H`` +``revalidated_decision`` composes the decision on that seam: it +reads a snapshot, computes the decision with the policy, then re-reads +the snapshot and compares full decision-input fingerprints until they +match — return values only: no status is read or written there. The +``publish --sha H`` subcommand consumes the seam's decision: it suppresses the write when the newest ``bits-review/head`` status created by ``dd-octo-sts[bot]`` already equals the decision, else POSTs it with ``STATUS_TOKEN`` only; @@ -51,6 +53,10 @@ #: Pages one paginated read (GraphQL reviews, REST lists) may follow; more #: is an error, never a truncation. MAX_FETCH_PAGES = 20 +#: Unstable fingerprint comparisons one decision may survive; after the +#: third nothing is written — a still-changing review set is not a +#: publishable state. +MAX_UNSTABLE_COMPARISONS = 3 #: Upper bound for honoring a server-provided rate-limit window #: (Retry-After, else x-ratelimit-reset), so a primary limit cannot stall #: the workflow run. @@ -548,20 +554,34 @@ def revalidated_decision(sha, client): """The validated decision for one head SHA: the contract's snapshot algorithm minus the write step. - Reads one snapshot through ``read_sha_snapshot`` (P(H) plus the - normalized reviews and error flags) and computes the decision with - the policy. Return values only, no side effects: no status is read + Reads S(i) through ``read_sha_snapshot`` (P(H) plus the normalized + reviews and error flags), computes D(i) with the policy, then reads + S(i+1) and compares ``policy.fingerprint`` over every decision + input. A mismatch discards D(i) — the loop restarts from the new + snapshot — and after ``MAX_UNSTABLE_COMPARISONS`` consecutive + mismatches returns ``("unstable", None)``: nothing may be + published. Return values only, no side effects: no status is read or written here — ``_publish_status`` does that on the returned decision. - - ``("empty", None)`` — P(H) was read successfully and is empty. - - ``("stable", D)`` — D is the decision of the snapshot. + - ``("empty", None)`` — P(H) was read successfully and is empty; + no second snapshot is read. + - ``("stable", D)`` — D is the decision of the revalidated + snapshot. + - ``("unstable", None)`` — the snapshot kept changing under the + reads. """ snapshot = read_sha_snapshot(sha, client) - decision = _decision_payload(snapshot, client.repo) - if decision is None: - return ("empty", None) - return ("stable", decision) + for _ in range(MAX_UNSTABLE_COMPARISONS): + decision = _decision_payload(snapshot, client.repo) + if decision is None: + return ("empty", None) + expected = policy.fingerprint(snapshot) + confirmed = read_sha_snapshot(sha, client) + if policy.fingerprint(confirmed) == expected: + return ("stable", decision) + snapshot = confirmed + return ("unstable", None) def _selected_app_status(statuses): @@ -621,8 +641,10 @@ def _publish_status(sha, client, status_token): """Publish the validated decision for one head SHA and return the process exit code. - The seam's ``empty`` return exits 0 without a write. For a decision - the latest app status is + The seam's ``empty`` return exits 0 without a write and its + ``unstable`` return exits 1 without a write — a still-changing + review set is never published. For a stable decision the latest app + status is selected and compared: an exact match skips the POST (exit 0 for a non-error decision — an ``error`` decision still exits 1 even though its POST is suppressed), and a failed status read never suppresses @@ -634,6 +656,8 @@ def _publish_status(sha, client, status_token): kind, decision = revalidated_decision(sha, client) if kind == "empty": return 0 + if kind == "unstable": + return 1 try: statuses = client.read_statuses(sha) except ApiError: diff --git a/.github/scripts/tests/test_bits_review_gate_publish.py b/.github/scripts/tests/test_bits_review_gate_publish.py index d3307f9a75..dfb8f30815 100644 --- a/.github/scripts/tests/test_bits_review_gate_publish.py +++ b/.github/scripts/tests/test_bits_review_gate_publish.py @@ -21,7 +21,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) import bits_review_gate as gate # noqa: E402 -import bits_review_gate_policy as policy # noqa: E402 import bits_review_gate_testlib as testlib # noqa: E402 @@ -176,10 +175,10 @@ def statuses_read_url(sha=HEAD): def publish_script(decision_snapshot, status_pages=(), post_response=None): - """The transport queue of one full publish run: the decision's single - snapshot, then every status page, then the POST response when a write - is expected.""" - responses = decision_snapshot[0] + list(status_pages) + """The transport queue of one full publish run: the decision's + revalidated snapshot pair, then every status page, then the POST + response when a write is expected.""" + responses = decision_snapshot[0] + decision_snapshot[0] + list(status_pages) if post_response is not None: responses.append(post_response) return responses @@ -219,11 +218,13 @@ def test_empty_p_h_returns_empty_from_one_snapshot_read(self): class StableDecisionTest(SeamTestCase): - """Snapshots return the exact decision payload of the single read; - D is asserted exactly and the one-snapshot read is pinned.""" + """A stable snapshot pair returns the exact decision payload of the + revalidated read; D is asserted exactly and the two-snapshot read is + pinned.""" def test_stable_snapshot_returns_the_exact_success_payload(self): - result, _ = self.run_seam(snapshot_script([member()], {968: [pass_node()]})) + script = snapshot_script([member()], {968: [pass_node()]}) + result, _ = self.run_seam(script, script) self.assertEqual(result[0], "stable") self.assertEqual(result[1], { "state": "success", @@ -240,13 +241,15 @@ def test_non_decision_input_fields_do_not_change_the_decision(self): changed = member() changed["title"] = "renamed after the read" changed["draft"] = True - result, transport = self.run_seam(snapshot_script([changed], {968: [pass_node()]})) + result, transport = self.run_seam(snapshot_script([changed], {968: [pass_node()]}), + snapshot_script([changed], {968: [pass_node()]})) self.assertEqual(result[0], "stable") self.assertEqual(result[1]["state"], "success") - self.assertEqual(len(transport.calls), 2) + self.assertEqual(len(transport.calls), 4) def test_stable_fail_payload_names_the_governing_review(self): - result, _ = self.run_seam(snapshot_script([member(985)], {985: [fail_node()]})) + script = snapshot_script([member(985)], {985: [fail_node()]}) + result, _ = self.run_seam(script, script) self.assertEqual(result[0], "stable") self.assertEqual(result[1], { "state": "failure", @@ -260,7 +263,8 @@ def test_stable_fail_payload_names_the_governing_review(self): def test_shared_head_pair_with_one_pass_and_one_missing_is_pending(self): prs = [member(968), member(969)] reviews = {968: [pass_node()], 969: []} - result, _ = self.run_seam(snapshot_script(prs, reviews)) + script = snapshot_script(prs, reviews) + result, _ = self.run_seam(script, script) self.assertEqual(result[0], "stable") self.assertEqual(result[1], { "state": "pending", @@ -275,12 +279,14 @@ def test_success_requires_every_member_to_pass(self): both_pass = [member(968), member(969)] reviews = {968: [pass_node()], 969: [testlib.fixture_node("pr968", databaseId=PASS_REVIEW_ID + 1)]} - result, _ = self.run_seam(snapshot_script(both_pass, reviews)) + script = snapshot_script(both_pass, reviews) + result, _ = self.run_seam(script, script) self.assertEqual((result[0], result[1]["state"]), ("stable", "success")) self.assertEqual(result[1]["governing_pr"], 968) pass_and_fail = [member(968), member(985)] reviews = {968: [pass_node()], 985: [fail_node()]} - result, _ = self.run_seam(snapshot_script(pass_and_fail, reviews)) + script = snapshot_script(pass_and_fail, reviews) + result, _ = self.run_seam(script, script) self.assertEqual(result[1]["state"], "failure") self.assertNotEqual(result[1]["reason"], "pass") @@ -295,7 +301,8 @@ def test_invalid_input_review_gives_a_stable_error_decision(self): databaseId=5873000001, submittedAt="2026-11-12T12:00:00Z", commit={"oid": HEAD}, body=marked_body, lastEditedAt="not-a-timestamp") reviews = {968: [pass_node(), broken]} - result, _ = self.run_seam(snapshot_script([member()], reviews)) + script = snapshot_script([member()], reviews) + result, _ = self.run_seam(script, script) self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), ("stable", "error", "invalid-input")) self.assertEqual(result[1]["description"], @@ -305,20 +312,21 @@ def test_malformed_pr_record_forces_error_over_clean_members(self): malformed = testlib.build_pr_record(number=7, sha="not-hex") prs = [member(), malformed] reviews = {968: [pass_node()]} - result, transport = self.run_seam(snapshot_script(prs, reviews)) + script = snapshot_script(prs, reviews) + result, transport = self.run_seam(script, script) self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), ("stable", "error", "invalid-input")) self.assertEqual(result[1]["governing_pr"], 968) self.assertEqual(result[1]["description"], "Bits review invalid-input on PR #968: pr-record:head.sha") self.assertEqual(result[1]["target_url"], PR968_URL) - self.assertEqual(len(transport.calls), 2) + self.assertEqual(len(transport.calls), 4) def test_failed_pulls_read_gives_a_stable_member_less_error_decision(self): """No PR could be scoped at all: the decision is still an error — never empty (the read did not succeed) — and, with no API PR number in the snapshot, description and target URL stay None.""" - result, transport = self.run_seam(failed_pulls_script()) + result, transport = self.run_seam(failed_pulls_script(), failed_pulls_script()) self.assertEqual(result[0], "stable") self.assertEqual(result[1], { "state": "error", @@ -328,7 +336,7 @@ def test_failed_pulls_read_gives_a_stable_member_less_error_decision(self): "target_url": None, "context": "bits-review/head", }) - self.assertEqual(len(transport.calls), 1) + self.assertEqual(len(transport.calls), 2) def test_pr_record_level_flag_outranks_a_member_api_error_in_reason_attribution(self): # The reason names the class of the flag that forced the level: a @@ -337,7 +345,7 @@ def test_pr_record_level_flag_outranks_a_member_api_error_in_reason_attribution( malformed = testlib.build_pr_record(number=7, sha="not-hex") prs = [member(), malformed] script = snapshot_script(prs, {968: failed_reviews_page()}) - result, _ = self.run_seam(script) + result, _ = self.run_seam(script, script) self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), ("stable", "error", "invalid-input")) self.assertEqual(result[1]["governing_pr"], 968) @@ -345,12 +353,148 @@ def test_pr_record_level_flag_outranks_a_member_api_error_in_reason_attribution( "Bits review invalid-input on PR #968: pr-record:head.sha") +class FingerprintChangeLoopTest(SeamTestCase): + """Every decision-input class changed between S(i) and S(i+1) forces a + third snapshot read and never returns the stale D(i): the returned + decision is recomputed from the changed snapshot.""" + + def loop_result(self, first, changed): + """S1, the changed S2 twice: three snapshots, six reads.""" + return self.run_seam(first, changed, changed) + + def loop_cases(self): + """(name, S2 open_prs, S2 reviews, expected decision asserts): + every fingerprint input class mutated away from the pass + snapshot.""" + edited_pass = testlib.fixture_node("pr968", + lastEditedAt="2026-11-12T13:00:00Z") + dismissed_pass = testlib.fixture_node("pr968", state="DISMISSED") + unlisted_author = testlib.fixture_node("pr968", author={ + "__typename": "Bot", "login": "other[bot]", "databaseId": 123456}) + user_author = testlib.fixture_node("pr968", author={ + "__typename": "User", "login": "human", "databaseId": 88084959}) + changed_login = testlib.fixture_node("pr968", author={ + "__typename": "Bot", "login": "renamed[bot]", "databaseId": 88084959}) + changed_commit = testlib.fixture_node("pr968", commit={"oid": "2" * 40}) + changed_submitted = testlib.fixture_node( + "pr968", submittedAt="2026-11-12T10:31:55Z") + new_review_id = testlib.fixture_node("pr968", databaseId=PASS_REVIEW_ID + 1) + return [ + ("base-ref", [member(base_ref="develop")], {968: [pass_node()]}, + ("success", 968)), + ("pr-state", [member(state="closed")], {968: [pass_node()]}, + ("success", 968)), + ("new-fail-review", [member()], {968: [fail_node()]}, + ("failure", 968)), + ("edited-pass", [member()], {968: [edited_pass]}, + ("failure", 968)), + ("dismissed-pass", [member()], {968: [dismissed_pass]}, + ("failure", 968)), + ("unlisted-author", [member()], {968: [unlisted_author]}, + ("failure", 968)), + ("user-author", [member()], {968: [user_author]}, + ("pending", 968)), + ("author-login", [member()], {968: [changed_login]}, + ("success", 968)), + ("commit-oid", [member()], {968: [changed_commit]}, + ("pending", 968)), + ("submitted-at", [member()], {968: [changed_submitted]}, + ("success", 968)), + ("new-review-id", [member()], {968: [new_review_id]}, + ("success", 968)), + ] + + def test_every_input_class_loops_and_never_returns_the_stale_decision(self): + first = snapshot_script([member()], {968: [pass_node()]}) + for name, prs, reviews, (state, governing) in self.loop_cases(): + with self.subTest(case=name): + changed = snapshot_script(prs, reviews) + result, transport = self.loop_result(first, changed) + self.assertEqual(result[0], "stable") + self.assertEqual(result[1]["state"], state) + if governing is not None: + self.assertEqual(result[1]["governing_pr"], governing) + # Three snapshots: the stale S1 never survives as the + # decision, and the loop revalidated S2 against S3. + self.assertEqual(len(transport.calls), 6) + + def test_edited_pass_never_publishes_a_stale_success(self): + """The gate's own threat: an edit landing between S1 and S2 turns + the published decision into the edited-summary failure.""" + edited = testlib.fixture_node("pr968", lastEditedAt="2026-11-12T13:00:00Z") + first = snapshot_script([member()], {968: [pass_node()]}) + changed = snapshot_script([member()], {968: [edited]}) + result, _ = self.loop_result(first, changed) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "failure", "edited-summary")) + + def test_member_head_moving_away_empties_the_snapshot(self): + """The member's head no longer matches the published SHA: the + changed snapshot is an empty P(H), returned after two reads.""" + first = snapshot_script([member()], {968: [pass_node()]}) + moved = snapshot_script([member(sha=OTHER_HEAD)], {}) + result, transport = self.run_seam(first, moved) + self.assertEqual(result, ("empty", None)) + self.assertEqual(len(transport.calls), 3) + + def test_membership_added_loops_and_conjoins_the_new_member(self): + """A second PR sharing the head appears in S2: the decision is + recomputed over both members, never the single-member stale D.""" + first = snapshot_script([member()], {968: [pass_node()]}) + both = [member(968), member(969)] + changed = snapshot_script(both, {968: [pass_node()], 969: []}) + result, transport = self.loop_result(first, changed) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "pending", "missing")) + self.assertEqual(result[1]["governing_pr"], 969) + # S1 reads one member (2 calls); S2 and S3 read two (3 calls each). + self.assertEqual(len(transport.calls), 8) + + def test_api_error_appearing_in_s2_loops_to_the_error_decision(self): + first = snapshot_script([member()], {968: [pass_node()]}) + changed = snapshot_script([member()], {968: failed_reviews_page()}) + result, _ = self.loop_result(first, changed) + self.assertEqual((result[0], result[1]["state"]), ("stable", "error")) + + def test_invalid_input_appearing_in_s2_loops_to_the_error_decision(self): + broken = testlib.build_review_node( + databaseId=5873000001, submittedAt="2026-11-12T12:00:00Z", + commit={"oid": HEAD}, body=testlib.load_review_fixture("pr985")["body"], + lastEditedAt="not-a-timestamp") + first = snapshot_script([member()], {968: [pass_node()]}) + changed = snapshot_script([member()], {968: [broken]}) + result, _ = self.loop_result(first, changed) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "error", "invalid-input")) + + +class UnstableSnapshotTest(SeamTestCase): + """Three consecutive fingerprint mismatches give up: (\"unstable\", + None) after exactly four snapshot reads.""" + + def test_three_mismatches_return_unstable_after_four_snapshots(self): + edited = testlib.fixture_node("pr968", lastEditedAt="2026-11-12T13:00:00Z") + dismissed = testlib.fixture_node("pr968", state="DISMISSED") + unlisted = testlib.fixture_node("pr968", author={ + "__typename": "Bot", "login": "other[bot]", "databaseId": 123456}) + scripts = [snapshot_script([member()], {968: [pass_node()]}), + snapshot_script([member()], {968: [edited]}), + snapshot_script([member()], {968: [dismissed]}), + snapshot_script([member()], {968: [unlisted]})] + result, transport = self.run_seam(*scripts) + self.assertEqual(result, ("unstable", None)) + self.assertEqual(len(transport.calls), 8) + for call in transport.calls: + self.assertNotIn("/statuses", call.full_url) + + class ZeroWriteAttemptTest(SeamTestCase): """The seam itself never reads or writes commit statuses: it returns values only, and every transport call stays on the two read URLs.""" def test_no_case_ever_touches_the_statuses_endpoint(self): - _, transport = self.run_seam(snapshot_script([member()], {968: [pass_node()]})) + script = snapshot_script([member()], {968: [pass_node()]}) + _, transport = self.run_seam(script, script) self.assertTrue(transport.calls) for call in transport.calls: self.assertNotIn("/statuses", call.full_url) @@ -434,8 +578,8 @@ def test_rate_limited_pulls_read_still_fails_closed_and_writes(self): self.assert_write_tokens(transport) class SeamExitMappingTest(PublishTestCase): - """The seam's empty return maps to exit 0 with zero write calls — and - no status read at all.""" + """The seam's empty return maps to exit 0 and its unstable return to + exit 1, both with zero write calls — and no status read at all.""" def test_empty_p_h_exits_0_with_no_status_read_or_write(self): script = snapshot_script([member(969, sha=OTHER_HEAD)], {})[0] @@ -446,6 +590,25 @@ def test_empty_p_h_exits_0_with_no_status_read_or_write(self): for call in transport.calls: self.assertNotIn("/statuses", call.full_url) + def test_unstable_snapshots_exit_1_with_no_status_read_or_write(self): + edited = testlib.fixture_node("pr968", lastEditedAt="2026-11-12T13:00:00Z") + dismissed = testlib.fixture_node("pr968", state="DISMISSED") + unlisted = testlib.fixture_node("pr968", author={ + "__typename": "Bot", "login": "other[bot]", "databaseId": 123456}) + snapshots = [snapshot_script([member()], {968: [pass_node()]}), + snapshot_script([member()], {968: [edited]}), + snapshot_script([member()], {968: [dismissed]}), + snapshot_script([member()], {968: [unlisted]})] + responses = [] + for responses_part, _ in snapshots: + responses.extend(responses_part) + code, transport = self.run_publish(responses) + self.assertEqual(code, 1) + self.assertEqual(len(transport.calls), 8) + self.assertEqual(self.post_calls(transport), []) + for call in transport.calls: + self.assertNotIn("/statuses", call.full_url) + class SuppressionTest(PublishTestCase): """Suppression compares only the selected latest app status; an @@ -456,7 +619,7 @@ def test_unchanged_success_skips_the_post_and_exits_0(self): pass_snapshot(), [FakeResponse(200, [status_record()])])) self.assertEqual(code, 0) self.assertEqual(self.post_calls(transport), []) - self.assertEqual(len(transport.calls), 3) + self.assertEqual(len(transport.calls), 5) def test_unchanged_error_skips_the_post_and_still_exits_1(self): statuses = FakeResponse(200, [status_record( From 8679b61455b4d0d1adf5d8ff8c2663f1f370698c Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 15:39:26 +0100 Subject: [PATCH 05/11] Retry bits-gate response-body read failures through the transport handler A timeout or reset during response.read() now falls into the existing retry and ApiError conversion instead of escaping the fetch loop; the best-effort summary-write swallow gets the comment the linter wants. --- .github/scripts/bits_review_gate.py | 14 ++++--- .../tests/test_bits_review_gate_api.py | 42 +++++++++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index 2b8ba905b7..82ed9e1ba5 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -334,6 +334,12 @@ def _fetch(self, url, payload=None, token=None): window = None try: response = self._transport(request) + try: + status = response.status + headers = getattr(response, "headers", None) or {} + body = response.read() + finally: + _close_quietly(response) except urllib.error.HTTPError as error: _close_quietly(error) if error.code < 500 and not _is_rate_limited(error.code, error.headers): @@ -345,12 +351,6 @@ def _fetch(self, url, payload=None, token=None): except Exception as error: # any transport failure retries failure = error else: - status = response.status - headers = getattr(response, "headers", None) or {} - try: - body = response.read() - finally: - _close_quietly(response) if 200 <= status < 300: return body, headers.get("Link") failure = ApiError("HTTP %s for %s" % (status, url)) @@ -766,6 +766,8 @@ def _note_unmatched_summary(env, selector, numbers): handle.write("bits gate: workflow_run selectors matched no open-main PR; " "publishing the run head only (numbers=%r)\n" % (numbers,)) except OSError: + # Best effort by the contract above: a failed summary write must + # not change the exit code or stdout. pass diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index b5eaa65527..68524bff48 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -446,6 +446,38 @@ def test_non_integer_retry_after_falls_back_to_default_backoff(self): [unittest.mock.call(gate._RETRY_BACKOFF_SECONDS)]) +class BodyReadRetryTest(unittest.TestCase): + """A transport failure during response-body read goes through the + same retry and ApiError conversion as the request itself — it must + never escape the transport handler.""" + + def test_body_read_failure_is_retried_until_success(self): + class FailingRead(FakeResponse): + def __init__(self, failures): + super().__init__(200, []) + self.failures = failures + + def read(self): + if self.failures: + self.failures -= 1 + raise OSError("connection reset during body read") + return self._body + + transport = testlib.FakeTransport([FailingRead(1), FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertEqual(len(transport.calls), 2) + + def test_body_read_failure_exhausts_attempts_and_fails_closed(self): + class AlwaysFailingRead(FakeResponse): + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport([AlwaysFailingRead(200, [])] * 3) + with self.assertRaises(gate.ApiError): + make_client(transport).read_statuses(SHA) + self.assertEqual(len(transport.calls), 3) + + class ResponseClosingTest(unittest.TestCase): """Every response opened in _fetch is closed on every exit path.""" @@ -475,6 +507,16 @@ def test_retried_http_error_closes_the_error(self): self.assertEqual(make_client(transport).list_open_main_prs(), []) self.assertTrue(error.closed) + def test_failed_body_read_still_closes_the_response(self): + class FailingRead(ClosingResponse): + def read(self): + raise OSError("connection reset during body read") + + response = FailingRead(200, []) + transport = testlib.FakeTransport([response, FakeResponse(200, [])]) + self.assertEqual(make_client(transport).list_open_main_prs(), []) + self.assertTrue(response.closed) + class MatchesDecisionTest(unittest.TestCase): """The suppression predicate requires every write-payload field to From 2808034380bf628dc5f4134bcadd285bc187f2de Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 15:46:49 +0100 Subject: [PATCH 06/11] Keep the bits gate one-attempt contract for 4xx body-read failures A non-retryable non-2xx whose body read fails raises ApiError after one attempt instead of retrying like a transport failure; a 2xx body read keeps retrying. Pins the revalidation flicker case (S1 pass, S2 edited, S3 pass again) returning the last revalidated decision. --- .github/scripts/bits_review_gate.py | 9 +++++++ .../tests/test_bits_review_gate_api.py | 26 +++++++++++++++++++ .../tests/test_bits_review_gate_publish.py | 13 ++++++++++ 3 files changed, 48 insertions(+) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index 82ed9e1ba5..62e722d25a 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -332,6 +332,8 @@ def _fetch(self, url, payload=None, token=None): for attempt in range(1, MAX_ATTEMPTS + 1): request = self._build_request(url, payload, token) window = None + status = None + headers = {} try: response = self._transport(request) try: @@ -349,6 +351,13 @@ def _fetch(self, url, payload=None, token=None): rate_limited_last = True window = _rate_limit_window_seconds(error.headers) except Exception as error: # any transport failure retries + if (status is not None + and not (200 <= status < 300 or status >= 500 + or _is_rate_limited(status, headers))): + # The status was already read as a non-retryable + # non-2xx; retrying a failed body read of it would + # break the one-attempt contract. + raise ApiError("HTTP %s for %s" % (status, url)) from error failure = error else: if 200 <= status < 300: diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index 68524bff48..635de4fd45 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -477,6 +477,32 @@ def read(self): make_client(transport).read_statuses(SHA) self.assertEqual(len(transport.calls), 3) + def test_non_retryable_4xx_body_read_failure_gets_exactly_one_attempt(self): + class FailingRead(FakeResponse): + def __init__(self, status): + super().__init__(status, []) + + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport([FailingRead(404)]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_retryable_5xx_body_read_failure_keeps_retrying(self): + class FailingRead(FakeResponse): + def __init__(self, status): + super().__init__(status, []) + + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport([FailingRead(503)] * 3) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 3) + class ResponseClosingTest(unittest.TestCase): """Every response opened in _fetch is closed on every exit path.""" diff --git a/.github/scripts/tests/test_bits_review_gate_publish.py b/.github/scripts/tests/test_bits_review_gate_publish.py index dfb8f30815..cd2f29e11f 100644 --- a/.github/scripts/tests/test_bits_review_gate_publish.py +++ b/.github/scripts/tests/test_bits_review_gate_publish.py @@ -428,6 +428,19 @@ def test_edited_pass_never_publishes_a_stale_success(self): self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), ("stable", "failure", "edited-summary")) + def test_flickering_input_returns_the_last_revalidated_decision(self): + """S1 pass, S2 edited, S3 and S4 pass again: every iteration + recomputes — the returned success is the final matched pair's, + never the intermediate edited failure nor a frozen earlier read.""" + edited = testlib.fixture_node("pr968", lastEditedAt="2026-11-12T13:00:00Z") + pass_script = snapshot_script([member()], {968: [pass_node()]}) + edited_script = snapshot_script([member()], {968: [edited]}) + result, transport = self.run_seam(pass_script, edited_script, + pass_script, pass_script) + self.assertEqual((result[0], result[1]["state"], result[1]["reason"]), + ("stable", "success", "pass")) + self.assertEqual(len(transport.calls), 8) + def test_member_head_moving_away_empties_the_snapshot(self): """The member's head no longer matches the published SHA: the changed snapshot is an empty P(H), returned after two reads.""" From f1d5a1177124246836db000d110432896d6628dc Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 17:38:21 +0100 Subject: [PATCH 07/11] Fail closed on non-object GraphQL roots in the bits gate A JSON null or [] root raised AttributeError inside the ApiError message expression instead of ApiError, terminating the publisher before the blocking status was written. The non-dict root now raises ApiError and flows into the error decision. --- .github/scripts/bits_review_gate.py | 4 +++- .../scripts/tests/test_bits_review_gate_api.py | 14 ++++++++++++++ .../tests/test_bits_review_gate_publish.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index 62e722d25a..8975193393 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -290,7 +290,9 @@ def _fetch_reviews_page(self, pr, cursor): document = json.loads(body) except ValueError as error: raise ApiError("malformed GraphQL response for PR %s" % (number,)) from error - if not isinstance(document, dict) or document.get("errors"): + if not isinstance(document, dict): + raise ApiError("malformed GraphQL response for PR %s" % (number,)) + if document.get("errors"): raise ApiError("GraphQL errors for PR %s: %r" % (number, document.get("errors"))) reviews = document for key in ("data", "repository", "pullRequest", "reviews"): diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index 635de4fd45..2d7c4e728b 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -580,6 +580,20 @@ def test_errors_with_partial_data_fail(self): make_client(transport).fetch_reviews(985) self.assertEqual(len(transport.calls), 1) + def test_non_object_root_fails_with_api_error_not_attribute_error(self): + # A JSON `null` or `[]` root must raise ApiError itself: the old + # message expression called document.get on the non-dict it had + # just detected and crashed with AttributeError instead. + for root in (None, []): + with self.subTest(root=root): + transport = testlib.FakeTransport( + [FakeResponse(200, json.dumps(root).encode("utf-8"))]) + with self.assertRaises(gate.ApiError) as raised: + make_client(transport).fetch_reviews(985) + self.assertIn("malformed GraphQL response for PR 985", + str(raised.exception)) + self.assertEqual(len(transport.calls), 1) + def test_missing_response_structure_fails(self): def drop_reviews(page): page["data"]["repository"]["pullRequest"].pop("reviews") diff --git a/.github/scripts/tests/test_bits_review_gate_publish.py b/.github/scripts/tests/test_bits_review_gate_publish.py index cd2f29e11f..32a15597e7 100644 --- a/.github/scripts/tests/test_bits_review_gate_publish.py +++ b/.github/scripts/tests/test_bits_review_gate_publish.py @@ -560,6 +560,23 @@ class RateLimitPublishTest(PublishTestCase): """Rate-limited reads failing past the honor cap publish diagnosable error decisions: the description names the rate-limit flag.""" + def test_null_graphql_root_publishes_the_blocking_error_status(self): + # A JSON `null` root must fail closed through ApiError into an + # error decision that is published — not crash the publisher with + # AttributeError before the blocking status is written. + null_root = FakeResponse(200, b"null") + script = snapshot_script([member()], {968: null_root}) + code, transport = self.run_publish(publish_script( + script, [FakeResponse(200, [])], FakeResponse(201, {}))) + self.assertEqual(code, 1) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + payload = json.loads(posts[0].data) + self.assertEqual(payload["state"], "error") + self.assertEqual(payload["description"], + "Bits review api-error on PR #968: pr-968/reviews") + self.assert_write_tokens(transport) + def test_rate_limited_reviews_read_names_the_rate_limit_in_the_description(self): # A reviews read failing past the honor cap flags the member as # pr-/reviews:rate-limit; the published description names it. From 1c41a568581a36ece1d70fde3c98c442281277a3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 17:46:36 +0100 Subject: [PATCH 08/11] Honor the rate-limit protocol on bits-gate body-read failures A rate-limited response whose body read fails now sets the rate-limit window and raises RateLimitError at exhaustion instead of retrying as a plain transport failure; the body-read classification boundaries (status 500 and 300) are pinned by tests. --- .github/scripts/bits_review_gate.py | 3 + .../tests/test_bits_review_gate_api.py | 64 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index 8975193393..0e185f8383 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -361,6 +361,9 @@ def _fetch(self, url, payload=None, token=None): # break the one-attempt contract. raise ApiError("HTTP %s for %s" % (status, url)) from error failure = error + if status is not None and _is_rate_limited(status, headers): + rate_limited_last = True + window = _rate_limit_window_seconds(headers) else: if 200 <= status < 300: return body, headers.get("Link") diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index 2d7c4e728b..a4131afda2 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -503,6 +503,70 @@ def read(self): make_client(transport).list_open_main_prs() self.assertEqual(len(transport.calls), 3) + def test_status_exactly_500_body_read_failure_still_retries(self): + # Pins the classification boundary: 500 is retryable, so a + # `>= 500` regression to `> 500` fails here. + class FailingRead(FakeResponse): + def __init__(self, status): + super().__init__(status, []) + + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport([FailingRead(500)] * 3) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 3) + + def test_status_exactly_300_body_read_failure_gets_one_attempt(self): + # Pins the other boundary: 300 is neither 2xx nor retryable, so a + # `< 300` regression to `<= 300` fails here. + class FailingRead(FakeResponse): + def __init__(self, status): + super().__init__(status, []) + + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport([FailingRead(300)]) + with self.assertRaises(gate.ApiError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + + def test_rate_limited_body_read_failure_raises_rate_limit_error(self): + # A rate-limited response whose body read fails must follow the + # rate-limit protocol — window honored, RateLimitError at + # exhaustion — not a generic transport ApiError. + class FailingRateLimitedRead(FakeResponse): + def __init__(self, status, headers): + super().__init__(status, []) + self.headers = headers + + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport( + [FailingRateLimitedRead(429, {"Retry-After": "0"})] * 3) + with self.assertRaises(gate.RateLimitError) as raised: + make_client(transport).list_open_main_prs() + self.assertIn("rate limit for", str(raised.exception)) + self.assertEqual(len(transport.calls), 3) + + def test_rate_limited_body_read_failure_honors_the_cap_and_fails_fast(self): + class FailingRateLimitedRead(FakeResponse): + def __init__(self, status, headers): + super().__init__(status, []) + self.headers = headers + + def read(self): + raise OSError("connection reset during body read") + + transport = testlib.FakeTransport( + [FailingRateLimitedRead(429, {"Retry-After": "600"})]) + with self.assertRaises(gate.RateLimitError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 1) + class ResponseClosingTest(unittest.TestCase): """Every response opened in _fetch is closed on every exit path.""" From c71c4813ba8c9f95c3a5fdb5f4e07f4499ad92c1 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 18:31:05 +0100 Subject: [PATCH 09/11] Classify bits-gate retry exhaustion by the last attempt rate_limited_last resets every attempt, so a mixed sequence that started rate-limited but ended in plain server errors reports a transport ApiError, not RateLimitError. --- .github/scripts/bits_review_gate.py | 5 ++++- .../tests/test_bits_review_gate_api.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index 0e185f8383..674bb72fbd 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -330,10 +330,13 @@ def _fetch(self, url, payload=None, token=None): rate-limit window beyond the honor cap fails fast instead of retrying. Returns (body_bytes, link_header).""" failure = None - rate_limited_last = False for attempt in range(1, MAX_ATTEMPTS + 1): request = self._build_request(url, payload, token) window = None + # The flag names the last attempt's classification: a mixed + # retry sequence must never report a rate limit its final + # failure did not have. + rate_limited_last = False status = None headers = {} try: diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index a4131afda2..62b3706ad7 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -435,6 +435,27 @@ def test_rate_limit_exhaustion_raises_a_named_rate_limit_error(self): str(caught.exception)) self.assertEqual(len(transport.calls), 3) + def test_mixed_sequence_ending_plain_reports_a_plain_api_error(self): + # The rate-limit classification names the LAST attempt: a run that + # started rate-limited but ended in plain 5xx failures is a + # transport failure, not a rate limit. + transport = testlib.FakeTransport([ + FakeStatusResponse(429, {"Retry-After": "0"}), + FakeResponse(500), FakeResponse(500)]) + with self.assertRaises(gate.ApiError) as caught: + make_client(transport).list_open_main_prs() + self.assertNotIsInstance(caught.exception, gate.RateLimitError) + self.assertEqual(len(transport.calls), 3) + + def test_mixed_sequence_ending_rate_limited_reports_rate_limit_error(self): + transport = testlib.FakeTransport([ + FakeResponse(500), + FakeStatusResponse(429, {"Retry-After": "0"}), + FakeStatusResponse(429, {"Retry-After": "0"})]) + with self.assertRaises(gate.RateLimitError): + make_client(transport).list_open_main_prs() + self.assertEqual(len(transport.calls), 3) + def test_non_integer_retry_after_falls_back_to_default_backoff(self): with unittest.mock.patch.object(gate.time, "sleep") as slept: transport = testlib.FakeTransport([ From 1af1e1aa4526843602840459983a3aa74c0c960b Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 19:00:57 +0100 Subject: [PATCH 10/11] Address the second review round on the bits gate - restore the contract's unmatched-selector fallback: a workflow_run matching no open-main PR resolves the run head plus every open-main head, not the run head alone; pin the one-variable base-change guard - read_statuses asks for per_page=100 so the 20-page cap fails at about 2000 statuses instead of 600 - the default transport refuses redirects, so the Authorization header (the status token on the POST) can never follow one to another host - resolve gets timeout-minutes 10 and publish 15: without a timeout they inherit the 360-minute default while _fetch can sleep 300 s per retry, and a stalled publish holds its per-SHA concurrency group - pin the unpinned suppression-comparison branches (state, target_url, foreign context) and drop RunnerDiscoveryTest's wrong AC-12 label --- .github/scripts/bits_review_gate.py | 33 +++++++---- .../tests/test_bits_review_gate_api.py | 58 +++++++++++++++++-- .../tests/test_bits_review_gate_publish.py | 17 +++++- .../tests/test_bits_review_gate_workflows.py | 13 ++++- .github/workflows/bits-review-gate.yml | 8 +++ 5 files changed, 111 insertions(+), 18 deletions(-) diff --git a/.github/scripts/bits_review_gate.py b/.github/scripts/bits_review_gate.py index 674bb72fbd..ddc6cf718d 100644 --- a/.github/scripts/bits_review_gate.py +++ b/.github/scripts/bits_review_gate.py @@ -110,8 +110,21 @@ class ResolveError(Exception): def _urllib_transport(request): - """The default transport: one stdlib urllib call per request.""" - return urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) + """The default transport: one stdlib urllib call per request, with + redirects refused — the redirect handler would forward the + Authorization header (the status token on the POST) to any target.""" + return _NO_REDIRECT_OPENER.open(request, timeout=_TIMEOUT_SECONDS) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Refuses every redirect: ``redirect_request`` returning None makes + urllib surface the 3xx as an HTTPError, which _fetch fails closed on.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +_NO_REDIRECT_OPENER = urllib.request.build_opener(_NoRedirect) def _next_link(link_header, base_url): @@ -243,7 +256,8 @@ def get_pr(self, number): def read_statuses(self, sha): """Every commit status recorded for ``sha``, all pages.""" - url = "%s/repos/%s/commits/%s/statuses" % (self._api_url, self._repo, sha) + url = "%s/repos/%s/commits/%s/statuses?per_page=%d" % ( + self._api_url, self._repo, sha, REST_PER_PAGE) return self._fetch_json_pages(url) def create_status(self, sha, status, token): @@ -781,7 +795,7 @@ def _note_unmatched_summary(env, selector, numbers): try: with open(path, "a", encoding="utf-8") as handle: handle.write("bits gate: workflow_run selectors matched no open-main PR; " - "publishing the run head only (numbers=%r)\n" % (numbers,)) + "scanning all open-main heads (numbers=%r)\n" % (numbers,)) except OSError: # Best effort by the contract above: a failed summary write must # not change the exit code or stdout. @@ -822,14 +836,11 @@ def resolve_shas(client, env): if pr["number"] in numbers or _pr_selector(pr) == selector] if matched: selected = matched - elif numbers: - # Selectors that match nothing publish the run head alone; the - # 30-minute schedule sweep recovers any missed head. - _note_unmatched_summary(env, selector, numbers) - selected = [] else: - # Fork runs carry no PR-number list; the full open-main - # fallback keeps every head monitored. + # Nothing matched — fork runs and stale number lists alike — + # so the full open-main fallback keeps every head monitored. + if numbers: + _note_unmatched_summary(env, selector, numbers) selected = open_prs shas = {head_sha} | {pr["head"]["sha"] for pr in selected} elif event in ("schedule", "workflow_dispatch"): diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index 62b3706ad7..ce6fba221e 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -34,7 +34,7 @@ PR985_HEAD = "1dcc3fa57582f2b595403f2b6a54e4464f352223" PR968_HEAD = "1115eb583f475b62544d634bb12bf750ae913e04" PULLS_URL = "%s/repos/%s/pulls?state=open&base=main&per_page=100" % (API, REPO) -STATUSES_URL = "%s/repos/%s/commits/%s/statuses" % (API, REPO, SHA) +STATUSES_URL = "%s/repos/%s/commits/%s/statuses?per_page=100" % (API, REPO, SHA) PR_URL = "%s/repos/%s/pulls/985" % (API, REPO) @@ -139,7 +139,7 @@ def test_page_without_next_link_ends_pagination(self): self.assertEqual(len(transport.calls), 1) def test_statuses_are_paginated_fully(self): - page2_url = STATUSES_URL + "?page=2" + page2_url = STATUSES_URL + "&page=2" transport = testlib.FakeTransport([ FakeResponse(200, [{"id": 1}], link='<%s>; rel="next"' % page2_url), FakeResponse(200, [{"id": 2}]), @@ -653,6 +653,32 @@ def test_foreign_context_never_matches(self): self.assertFalse(gate._matches_decision( self._status(context="bits-review/head-evil"), self._decision())) + def test_differing_state_never_matches(self): + self.assertFalse(gate._matches_decision( + self._status(state="failure"), self._decision())) + + def test_differing_target_url_never_matches(self): + self.assertFalse(gate._matches_decision( + self._status(target_url="https://example.test/other"), self._decision())) + + +class DefaultTransportTest(unittest.TestCase): + """The default transport must refuse redirects: urllib's redirect + handler would forward the Authorization header — the status token on + the POST — to whatever target answers, including another host.""" + + def test_no_redirect_handler_is_installed(self): + handlers = [type(handler) for handler in gate._NO_REDIRECT_OPENER.handlers] + self.assertIn(gate._NoRedirect, handlers) + + def test_redirect_request_refuses_every_redirect(self): + request = urllib.request.Request("https://api.example.test/repos/x") + for code in (301, 302, 303, 307, 308): + with self.subTest(code=code): + handler = gate._NoRedirect() + self.assertIsNone(handler.redirect_request( + request, None, code, "Moved", {}, "https://evil.example.test/x")) + class GraphqlFailClosedTest(unittest.TestCase): """GraphQL errors and missing structure never yield a partial list.""" @@ -1113,12 +1139,14 @@ def test_workflow_run_matches_by_head_repository_and_branch(self): self.assertEqual(code, 0) self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD])) - def test_workflow_run_non_matching_selector_publishes_the_run_head_only(self): + def test_workflow_run_non_matching_numbers_scan_all_open_main_heads(self): + # The contract's fallback: when nothing matches, the run head plus + # every open-main head is resolved — never the run head alone. prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] code, out = self._run(self._schedule(prs), "workflow_run", GITHUB_PULL_REQUEST_NUMBERS="[7]") self.assertEqual(code, 0) - self.assertEqual(json.loads(out), sorted([SHA])) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD, PR985_HEAD])) def test_workflow_run_number_list_with_null_padding_still_matches(self): prs = [open_main_pr(968, PR968_HEAD), open_main_pr(985, PR985_HEAD)] @@ -1139,7 +1167,7 @@ def test_summary_write_failure_leaves_exit_code_and_stdout_untouched(self): GITHUB_PULL_REQUEST_NUMBERS="[7]", GITHUB_STEP_SUMMARY=summary) self.assertEqual(code, 0) - self.assertEqual(json.loads(out), sorted([SHA])) + self.assertEqual(json.loads(out), sorted([SHA, PR968_HEAD, PR985_HEAD])) def test_output_write_failure_exits_one(self): # The shas output is load-bearing: an I/O failure there fails the @@ -1163,6 +1191,26 @@ def test_edited_with_a_base_change_resolves_normally(self): self.assertEqual(json.loads(out), [PR985_HEAD]) self.assertEqual([call.full_url for call in transport.calls], [PR_URL]) + def test_edited_with_only_a_base_ref_change_resolves_normally(self): + pr = testlib.build_pr_record(number=985, sha=PR985_HEAD, base_ref="develop") + transport = testlib.FakeTransport([FakeResponse(200, pr)]) + code, out = self._run(transport, "pull_request_target", + GITHUB_PR_ACTION="edited", + GITHUB_CHANGES_BASE_REF="main") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [PR985_HEAD]) + self.assertEqual([call.full_url for call in transport.calls], [PR_URL]) + + def test_edited_with_only_a_base_sha_change_resolves_normally(self): + pr = testlib.build_pr_record(number=985, sha=PR985_HEAD, base_ref="develop") + transport = testlib.FakeTransport([FakeResponse(200, pr)]) + code, out = self._run(transport, "pull_request_target", + GITHUB_PR_ACTION="edited", + GITHUB_CHANGES_BASE_SHA="b" * 40) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out), [PR985_HEAD]) + self.assertEqual([call.full_url for call in transport.calls], [PR_URL]) + def test_edited_without_a_base_change_resolves_to_nothing(self): # A title/body edit cannot change any decision input: the guard # returns an empty list with no API call at all. diff --git a/.github/scripts/tests/test_bits_review_gate_publish.py b/.github/scripts/tests/test_bits_review_gate_publish.py index 32a15597e7..02cb92f421 100644 --- a/.github/scripts/tests/test_bits_review_gate_publish.py +++ b/.github/scripts/tests/test_bits_review_gate_publish.py @@ -171,7 +171,7 @@ def rate_limited_response(status=429, headers=None): def statuses_read_url(sha=HEAD): - return "%s/repos/%s/commits/%s/statuses" % (API, REPO, sha) + return "%s/repos/%s/commits/%s/statuses?per_page=100" % (API, REPO, sha) def publish_script(decision_snapshot, status_pages=(), post_response=None): @@ -733,6 +733,21 @@ def test_malformed_next_link_fails_the_status_read_closed_and_still_writes(self) call.full_url for call in transport.calls)) self.assert_write_tokens(transport) + def test_foreign_context_status_is_never_selected_for_comparison(self): + # The newest app-creator status matches the decision in every + # compared field but its context is foreign: it is not this gate's + # output and must never suppress the write. + statuses = FakeResponse(200, [ + status_record(context="some-other-context", status_id=900, + created_at="2026-11-12T13:00:00Z"), + ]) + code, transport = self.run_publish(publish_script( + pass_snapshot(), [statuses], FakeResponse(201, {}))) + self.assertEqual(code, 0) + posts = self.post_calls(transport) + self.assertEqual(len(posts), 1) + self.assertEqual(json.loads(posts[0].data)["description"], PASS_DESCRIPTION) + def test_malformed_app_status_is_never_selected_for_comparison(self): # A malformed app-creator entry that matches the decision in every # compared field is never selected for the suppression comparison: diff --git a/.github/scripts/tests/test_bits_review_gate_workflows.py b/.github/scripts/tests/test_bits_review_gate_workflows.py index 0bc815673a..13afd1de71 100644 --- a/.github/scripts/tests/test_bits_review_gate_workflows.py +++ b/.github/scripts/tests/test_bits_review_gate_workflows.py @@ -239,6 +239,15 @@ def test_workflow_name(self): re.search(r"^name: Bits review gate$", self.text, re.M) ) + def test_both_jobs_carry_a_timeout(self): + # Without a timeout the jobs inherit the 360-minute default, and + # _fetch can sleep up to 300 s per retry — a stalled publish also + # holds its per-SHA concurrency group (cancel-in-progress false). + for block, name in ((self.resolve, "resolve"), (self.publish, "publish")): + with self.subTest(job=name): + self.assertIsNotNone( + re.search(r"^[ \t]+timeout-minutes: \d+$", block, re.M)) + def test_triggers_are_exactly_the_four_events(self): on_keys = set(re.findall(r"^ ([^\s:]+):", self.on, re.M)) self.assertEqual( @@ -535,7 +544,9 @@ def test_loop_keeps_existing_entries(self): class RunnerDiscoveryTest(unittest.TestCase): - """AC-12: the gate runner discovers only this gate's test family.""" + """The gate runner discovers only this gate's test family (INV-22); + the design doc's AC12 is the Octo-STS claim-denial test drive, not + this local runner contract.""" @classmethod def setUpClass(cls): diff --git a/.github/workflows/bits-review-gate.yml b/.github/workflows/bits-review-gate.yml index 2a07c97503..20fd47c21b 100644 --- a/.github/workflows/bits-review-gate.yml +++ b/.github/workflows/bits-review-gate.yml @@ -18,6 +18,10 @@ jobs: # is live for it too; the other triggers already run on refs/heads/main. if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest + # Without this the job inherits the 360-minute default; _fetch can + # sleep up to 300 s per retry, so a stalled run would hold the + # workflow for hours. + timeout-minutes: 10 permissions: contents: read pull-requests: read @@ -48,6 +52,10 @@ jobs: needs: resolve if: github.ref == 'refs/heads/main' && needs.resolve.outputs.shas != '[]' runs-on: ubuntu-latest + # The per-SHA concurrency group does not cancel stalled runs + # (cancel-in-progress: false), so a hung publish would delay every + # later update for this SHA until the timeout fires. + timeout-minutes: 15 permissions: contents: read pull-requests: read From ddeb78e892ad88227c86a1e3f8045ae16576d9a0 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 13 Nov 2026 19:14:20 +0100 Subject: [PATCH 11/11] Pin the bits gate default transport to the no-redirect opener The two DefaultTransportTest tests only inspect the module objects, so reverting _urllib_transport to plain urlopen left them green while reintroducing redirect-token forwarding. The new test drives the transport through a recording opener swapped into _NO_REDIRECT_OPENER and fails on that revert (verified by mutation). --- .../tests/test_bits_review_gate_api.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/scripts/tests/test_bits_review_gate_api.py b/.github/scripts/tests/test_bits_review_gate_api.py index ce6fba221e..b32473ccd8 100644 --- a/.github/scripts/tests/test_bits_review_gate_api.py +++ b/.github/scripts/tests/test_bits_review_gate_api.py @@ -679,6 +679,28 @@ def test_redirect_request_refuses_every_redirect(self): self.assertIsNone(handler.redirect_request( request, None, code, "Moved", {}, "https://evil.example.test/x")) + def test_transport_goes_through_the_no_redirect_opener(self): + # Pins the wiring the guarded property depends on: reverting + # _urllib_transport to plain urlopen leaves the two tests above + # green while reintroducing redirect-token forwarding, so this + # one must fail on that revert. + recorded = [] + + class RecordingOpener: + def open(self, request, timeout=None): + recorded.append((request, timeout)) + return object() + + original = gate._NO_REDIRECT_OPENER + gate._NO_REDIRECT_OPENER = RecordingOpener() + try: + request = urllib.request.Request("https://api.example.test/repos/x") + gate._urllib_transport(request) + finally: + gate._NO_REDIRECT_OPENER = original + self.assertEqual(len(recorded), 1) + self.assertEqual(recorded[0][1], gate._TIMEOUT_SECONDS) + class GraphqlFailClosedTest(unittest.TestCase): """GraphQL errors and missing structure never yield a partial list."""