Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions scripts/check_codex_comments_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,24 @@ def test_metadata_is_not_approval(self):
result = self.run_gate(
snapshot([request, comment(fixture["body"])]), "wait_pr_codex.sh"
)
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
# Completed informational envelopes keep polling for approval;
# unfinished/unknown reports remain failures under the CI policy.
expected = 10 if fixture["expected_exit_code"] == 0 else 1
self.assertEqual(result.returncode, expected, result.stdout + result.stderr)

def test_informational_comments_do_not_hide_findings_or_account_errors(self):
request = comment("@codex review", "maintainer", REQUEST)
for name in ("summary", "security_no_findings"):
comments = [request, comment(FIXTURES[name]["body"])]
for extra_comments, threads in (
([comment("[P1] Fix authorization")], []),
([], [thread("[P1] Fix authorization")]),
([comment("Please create a Codex account to review.")], []),
):
with self.subTest(name=name, comments=extra_comments, threads=threads):
self.assert_gate(
1, snapshot(comments + extra_comments, threads), "wait_pr_codex.sh"
)

def test_only_authenticated_codex_authors_get_protocol_exemptions(self):
comments = [
Expand Down Expand Up @@ -275,8 +292,8 @@ def test_only_authenticated_codex_authors_get_protocol_exemptions(self):
def test_approval_requires_the_real_bot_and_fresh_signal(self):
request = comment("@codex review", "maintainer", REQUEST)
for author, created_at, expected in (
(BOT, BEFORE, 1),
("human-reviewer", AFTER, 1),
(BOT, BEFORE, 10),
("human-reviewer", AFTER, 10),
(BOT, AFTER, 0),
):
with self.subTest(author=author, created_at=created_at):
Expand Down
12 changes: 11 additions & 1 deletion scripts/wait_pr_codex.sh
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,17 @@ CHECK_CODEX_STATUS_ONCE() {
return 0
fi

codex_response_count_comments=$(echo "$all_comments" | jq -r --arg bot "$BOT_LOGIN_GRAPHQL" --arg request_at "$request_at" '[.[] | select(.author.login == $bot and .createdAt > $request_at)] | length')
# Completed status/no-findings envelopes are neither approval nor a failed
# review. Reuse the CI classifier and keep waiting for the approval signal.
# Unfinished/unknown envelopes and account errors retain their blocking behavior.
codex_response_count_comments=$(echo "$all_comments" | jq -r -L "$SCRIPT_DIR/lib" --arg bot "$BOT_LOGIN_GRAPHQL" --arg request_at "$request_at" '
include "codex_comments";
[.[] | select(.author.login == $bot and .createdAt > $request_at)
| select(
((.body | startswith("<!-- codex-pull-request-review-summary -->") or startswith("Security review completed."))
and codex_comment_is_informational($bot)) | not
)] | length
')
codex_response_count_threads=$(echo "$all_threads" | jq -r --arg bot "$BOT_LOGIN_GRAPHQL" --arg request_at "$request_at" '[.[] | select((.comments.nodes | length) > 0 and .comments.nodes[0].author.login == $bot and .comments.nodes[0].createdAt > $request_at)] | length')
codex_response_count=$((codex_response_count_comments + codex_response_count_threads))

Expand Down
Loading