diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 6d8f9102..deed037d 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -900,9 +900,7 @@ jobs: check_javascript_coverage_thresholds() { local summary_list - local checker summary_list="${RUNNER_TEMP}/javascript-coverage-summaries.txt" - checker="${RUNNER_TEMP}/check-javascript-coverage.py" find . \ \( -path '*/coverage/coverage-summary.json' -o -path '*/coverage/coverage-final.json' \) \ -type f \ @@ -919,106 +917,12 @@ jobs: return fi - cat >"$checker" <<'PY' - import json - import sys - from pathlib import Path - - def pct(covered: int, total: int) -> float: - return 100.0 if total == 0 else round((covered / total) * 100, 2) - - - def summarize_final(data: dict) -> dict[str, float]: - totals = { - "statements": [0, 0], - "branches": [0, 0], - "functions": [0, 0], - "lines": [0, 0], - } - for file_data in data.values(): - statements = file_data.get("s") or {} - totals["statements"][1] += len(statements) - totals["statements"][0] += sum(1 for count in statements.values() if count > 0) - - functions = file_data.get("f") or {} - totals["functions"][1] += len(functions) - totals["functions"][0] += sum(1 for count in functions.values() if count > 0) - - branches = file_data.get("b") or {} - for counts in branches.values(): - totals["branches"][1] += len(counts) - totals["branches"][0] += sum(1 for count in counts if count > 0) - - line_counts: dict[int, int] = {} - statement_map = file_data.get("statementMap") or {} - for statement_id, location in statement_map.items(): - start = (location.get("start") or {}).get("line") - if start is None: - continue - line_counts[start] = max(line_counts.get(start, 0), statements.get(statement_id, 0)) - totals["lines"][1] += len(line_counts) - totals["lines"][0] += sum(1 for count in line_counts.values() if count > 0) - - return { - metric: pct(values[0], values[1]) - for metric, values in totals.items() - } - - - summary_list = Path(sys.argv[1]) - failures: list[str] = [] - for raw_path in summary_list.read_text(encoding="utf-8").splitlines(): - summary_path = Path(raw_path) - data = json.loads(summary_path.read_text(encoding="utf-8")) - if summary_path.name == "coverage-summary.json": - metric_totals = { - metric: (data.get("total") or {}).get(metric, {}).get("pct") - for metric in ("statements", "branches", "functions", "lines") - } - else: - metric_totals = summarize_final(data) - print(f"{summary_path}:") - for metric in ("statements", "branches", "functions", "lines"): - metric_pct = metric_totals.get(metric) - print(f" {metric}: {metric_pct}%") - if metric_pct != 100: - failures.append(f"{summary_path} {metric}={metric_pct}%") - if summary_path.name == "coverage-summary.json": - for file_name, file_summary in sorted(data.items()): - if file_name == "total": - continue - below = [] - for metric in ("statements", "branches", "functions", "lines"): - metric_pct = (file_summary.get(metric) or {}).get("pct") - if metric_pct != 100: - below.append(f"{metric}={metric_pct}%") - if below: - print(f" file below 100%: {file_name} ({', '.join(below)})") - else: - for file_name, file_data in sorted(data.items()): - statements = file_data.get("s") or {} - statement_map = file_data.get("statementMap") or {} - missing_lines = [] - for statement_id, count in statements.items(): - if count > 0: - continue - start = (statement_map.get(statement_id) or {}).get("start") or {} - line = start.get("line") - if line is not None: - missing_lines.append(line) - if missing_lines: - line_list = ",".join(str(line) for line in sorted(set(missing_lines))[:60]) - suffix = "" if len(set(missing_lines)) <= 60 else ",..." - print(f" missing lines: {file_name}:{line_list}{suffix}") - - if failures: - print("Coverage below 100%:") - for failure in failures: - print(f"- {failure}") - raise SystemExit(1) - PY - - run_and_capture "JavaScript/TypeScript coverage threshold" python3 "$checker" "$summary_list" + run_and_capture "JavaScript/TypeScript coverage threshold" \ + python3 "$GITHUB_WORKSPACE/scripts/ci/javascript_coverage_gate.py" \ + --repo-root . \ + --base-sha "$PR_BASE_SHA" \ + --head-sha "$PR_HEAD_SHA" \ + --summary-list "$summary_list" } ensure_r_runtime() { @@ -1766,6 +1670,7 @@ jobs: ContextualWisdomLab/.github:code-reviewer-prompt.md | \ ContextualWisdomLab/.github:opencode.jsonc | \ ContextualWisdomLab/.github:scripts/ci/changed_file_syntax_gate.py | \ + ContextualWisdomLab/.github:scripts/ci/javascript_coverage_gate.py | \ ContextualWisdomLab/.github:scripts/ci/opencode_review_approve_gate.sh | \ ContextualWisdomLab/.github:scripts/ci/pr_head_replay_guard.py | \ ContextualWisdomLab/.github:scripts/ci/pr_review_merge_scheduler.py | \ @@ -1774,6 +1679,7 @@ jobs: ContextualWisdomLab/.github:scripts/ci/strix_quick_gate.sh | \ ContextualWisdomLab/.github:scripts/ci/validate_opencode_failed_check_review.sh | \ ContextualWisdomLab/.github:tests/test_changed_file_syntax_gate.py | \ + ContextualWisdomLab/.github:tests/test_javascript_coverage_gate.py | \ ContextualWisdomLab/.github:tests/test_opencode_agent_contract.py | \ ContextualWisdomLab/.github:tests/test_opencode_model_pool_runner.py | \ ContextualWisdomLab/.github:tests/test_pr_head_replay_guard.py | \ @@ -1850,6 +1756,14 @@ jobs: printf 'Fallback ineligibility reasons: none\n' fi + - name: Install central adversarial harness runtime + if: >- + needs.coverage-evidence.result == 'success' + && steps.central_review_process_fallback_scope.outputs.eligible == 'true' + run: >- + python3 -m pip install --disable-pip-version-check --require-hashes + --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt + - name: Initialize CodeGraph index for OpenCode env: CODEGRAPH_PACKAGE: "@colbymchenry/codegraph@0.9.9" @@ -3056,6 +2970,14 @@ jobs: "apiKey": "{env:STRIX_GITHUB_MODELS_TOKEN}" }, "models": { + "openai/gpt-4.1": { + "name": "OpenAI GPT-4.1", + "tool_call": true, + "limit": { + "context": 1048576, + "output": 32768 + } + }, "openai/gpt-5": { "name": "OpenAI GPT-5", "tool_call": true, @@ -3251,7 +3173,7 @@ jobs: - name: Run OpenCode PR Review model pool id: opencode_review_model_pool if: needs.coverage-evidence.result == 'success' - timeout-minutes: 20 + timeout-minutes: 12 continue-on-error: true env: STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} @@ -3269,12 +3191,13 @@ jobs: NO_COLOR: "1" # High-sensitivity review candidates only. DeepSeek V3 has been the # most reliable first-pass reviewer in the org queue, then the pool - # falls through to full-size GPT/o3 and reasoning-capable fallbacks. + # falls through to the direct GPT-5.6 Luna slot, then the full-size + # GPT-4.1 long-context endpoint and provider-specific GPT/o3 fallbacks. # The direct-OpenAI slot runs GPT-5.6 Luna: the newest family's # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. @@ -3282,10 +3205,10 @@ jobs: # Keep stale providers from pinning required review jobs for hours. # Adversarial validation needs enough room to read the evidence, but # dynamic cadence and the outer watchdog still bound each current-head run. - OPENCODE_RUN_TIMEOUT_SECONDS: "300" + OPENCODE_RUN_TIMEOUT_SECONDS: "180" OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "600" - OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "1080" + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540" + OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "540" # Visit the high-sensitivity candidate catalog once. Per-provider # failures and timeouts are logged before the publish gate evaluates # the current-head result and fails closed when no verdict exists. @@ -3293,14 +3216,19 @@ jobs: OPENCODE_DYNAMIC_REVIEW_CADENCE: "true" OPENCODE_SMALL_CHANGE_FILE_THRESHOLD: "3" OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD: "20" - OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "300" - OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "600" - OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "420" - OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "900" - OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "420" - OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "1080" - OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "420" - OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "900" + OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "90" + OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "180" + OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "120" + OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "360" + OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "180" + OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "540" + OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "120" + OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "360" + # This installation currently reports a 4k request-body limit for + # GitHub Models GPT-5 endpoints even though the public catalog is + # larger. Keep the exact runtime failure visible without spending a + # full medium/large cadence slot after the long-context candidate. + OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45" OPENCODE_DYNAMIC_MAX_CYCLES: "1" CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE: ${{ steps.central_review_process_fallback_scope.outputs.eligible || 'false' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} @@ -3326,13 +3254,13 @@ jobs: run: | set -euo pipefail set +e - timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-1080}s" \ + timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-540}s" \ bash "$GITHUB_WORKSPACE/scripts/ci/run_opencode_review_model_pool.sh" pool_status=$? set -e if [ "$pool_status" -eq 124 ] || [ "$pool_status" -eq 137 ] || [ "$pool_status" -eq 143 ]; then printf 'OpenCode model pool exceeded the outer %ss step budget; marking the pool exhausted so current-head evidence fallback can publish a bounded reason instead of blocking the org queue.\n' \ - "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-1080}" + "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-540}" { printf 'review_model=\n' printf 'review_status=exhausted\n' @@ -3417,8 +3345,9 @@ jobs: if: >- always() && steps.opencode_review_model_pool.outputs.review_status == 'success' + && steps.opencode_app_token.outputs.available == 'true' env: - GH_TOKEN: ${{ steps.opencode_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }} GH_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.inputs.target_repository || github.repository }} PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.inputs.pr_head_sha }} @@ -3449,9 +3378,10 @@ jobs: comment_body_file="$(mktemp)" normalized_comment_json="$(mktemp)" overview_body_file="$(mktemp)" + overview_response_file="$(mktemp)" gh_error_file="$(mktemp)" cleanup_publish_files() { - rm -f "$clean_output" "$comment_body_file" "$normalized_comment_json" "$overview_body_file" "$gh_error_file" + rm -f "$clean_output" "$comment_body_file" "$normalized_comment_json" "$overview_body_file" "$overview_response_file" "$gh_error_file" } trap cleanup_publish_files EXIT @@ -3668,23 +3598,42 @@ jobs: append_merge_conflict_guidance } >"$overview_body_file" + live_head="$(gh api -X GET "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha // empty' 2>"$gh_error_file" || true)" + if [ "$live_head" != "$HEAD_SHA" ]; then + printf '::error::OPENCODE_OVERVIEW_STALE_HEAD: refusing initial overview publication because expected head %s no longer matches live head %s.\n' "$HEAD_SHA" "${live_head:-missing}" + exit 1 + fi + + published_overview_comment_id="" if ! overview_comment_id="$( gh api -X GET "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq '[.[] | select((.user.login == "github-actions[bot]" or .user.login == "opencode-agent[bot]") and (.body | contains("")))] | sort_by(.created_at) | last.id // empty' \ + --jq '[.[] | select(.user.login == "opencode-agent[bot]" and (.body | contains("")))] | sort_by(.created_at) | last.id // empty' \ 2>"$gh_error_file" )"; then warn_gh_publication_failure "initial review overview lookup" "$gh_error_file" elif [ -n "$overview_comment_id" ]; then : >"$gh_error_file" if ! jq -n --rawfile body "$overview_body_file" '{body: $body}' | - gh api -X PATCH "repos/${GH_REPOSITORY}/issues/comments/${overview_comment_id}" --input - >/dev/null 2>"$gh_error_file"; then + gh api -X PATCH "repos/${GH_REPOSITORY}/issues/comments/${overview_comment_id}" --input - >"$overview_response_file" 2>"$gh_error_file"; then warn_gh_publication_failure "initial review overview update" "$gh_error_file" + else + published_overview_comment_id="$overview_comment_id" fi else : >"$gh_error_file" if ! jq -n --rawfile body "$overview_body_file" '{body: $body}' | - gh api -X POST "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --input - >/dev/null 2>"$gh_error_file"; then + gh api -X POST "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --input - >"$overview_response_file" 2>"$gh_error_file"; then warn_gh_publication_failure "initial review overview comment" "$gh_error_file" + else + published_overview_comment_id="$(jq -r '.id // empty' "$overview_response_file")" + fi + fi + if [ -n "$published_overview_comment_id" ]; then + live_head="$(gh api -X GET "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha // empty' 2>"$gh_error_file" || true)" + if [ "$live_head" != "$HEAD_SHA" ]; then + gh api -X DELETE "repos/${GH_REPOSITORY}/issues/comments/${published_overview_comment_id}" >/dev/null 2>>"$gh_error_file" || true + printf '::error::OPENCODE_OVERVIEW_STALE_HEAD: deleted initial overview after head advanced from %s to %s.\n' "$HEAD_SHA" "${live_head:-missing}" + exit 1 fi fi @@ -3695,9 +3644,10 @@ jobs: && needs.coverage-evidence.result == 'success' && steps.opencode_review_model_pool.outputs.review_status == 'success' && steps.central_review_process_fallback_scope.outputs.eligible == 'true' + continue-on-error: true timeout-minutes: 3 env: - GH_TOKEN: ${{ steps.opencode_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }} CHECK_LOOKUP_GH_TOKEN: ${{ github.token }} GH_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.inputs.target_repository || github.repository }} PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }} @@ -3781,6 +3731,22 @@ jobs: def self_check: (.name // "") as $n | ["opencode-review", "coverage-evidence", "coverage-source-tree", "required-workflow-bootstrap"] | index($n); + def latest_peer_checks: + [ + (.check_runs // [])[] + | select(self_check | not) + | . + { + checkedAt: ( + if ((.started_at // "") != "") then .started_at + else (.completed_at // "") + end + ) + } + ] + | sort_by(.app.slug // "", .name // "", .checkedAt // "", .id // 0) + | group_by([.app.slug // "", .name // ""]) + | map(last) + | .[]; ' check_runs_file="$(mktemp)" @@ -3789,9 +3755,8 @@ jobs: for attempt in $(seq 1 "${APPROVAL_CHECK_WAIT_ATTEMPTS:-12}"); do curl_api_read "${api_url}/repos/${GH_REPOSITORY}/commits/${HEAD_SHA}/check-runs?per_page=100" >"$check_runs_file" jq -r "${self_check_filter} - (.check_runs // []) - | .[] - | select((self_check | not) and (.status // \"\") != \"completed\") + latest_peer_checks + | select((.status // \"\") != \"completed\") | \"- \" + (.name // \"check\") + \": \" + (.status // \"unknown\") + (if (.html_url // \"\") != \"\" then \" (\" + .html_url + \")\" else \"\" end) " "$check_runs_file" >"$pending_checks_file" if [ ! -s "$pending_checks_file" ]; then @@ -3809,9 +3774,8 @@ jobs: exit 1 fi jq -r "${self_check_filter} - (.check_runs // []) - | .[] - | select((self_check | not) and (.status // \"\") == \"completed\") + latest_peer_checks + | select((.status // \"\") == \"completed\") | select((.conclusion // \"\") as \$c | [\"success\", \"neutral\", \"skipped\"] | index(\$c) | not) | \"- \" + (.name // \"check\") + \": \" + (.conclusion // \"unknown\") + (if (.html_url // \"\") != \"\" then \" (\" + .html_url + \")\" else \"\" end) " "$check_runs_file" >"$failed_checks_file" @@ -3897,9 +3861,33 @@ jobs: "" \ "This approval path is limited to ContextualWisdomLab/.github central review-process self-repair.")" payload_file="$(mktemp)" + live_head_file="$(mktemp)" + review_response_file="$(mktemp)" + dismissal_payload_file="$(mktemp)" jq -n --arg event APPROVE --arg body "$body" --arg commit_id "$HEAD_SHA" \ '{event: $event, body: $body, commit_id: $commit_id}' >"$payload_file" - curl_api_write -X POST --data-binary "@${payload_file}" "${api_url}/repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}/reviews" + curl_api_read "${api_url}/repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" >"$live_head_file" + live_head="$(jq -r '.head.sha // empty' "$live_head_file")" + if [ "$live_head" != "$HEAD_SHA" ]; then + echo "::notice::CENTRAL_FAST_APPROVAL_STALE_HEAD: expected ${HEAD_SHA}, observed ${live_head:-missing}; skipping review publication." + exit 0 + fi + curl_api_write -X POST --data-binary "@${payload_file}" \ + "${api_url}/repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}/reviews" >"$review_response_file" + curl_api_read "${api_url}/repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" >"$live_head_file" + live_head="$(jq -r '.head.sha // empty' "$live_head_file")" + if [ "$live_head" != "$HEAD_SHA" ]; then + review_id="$(jq -r '.id // empty' "$review_response_file")" + review_state="$(jq -r '(.state // "") | ascii_upcase' "$review_response_file")" + if [ -n "$review_id" ] && { [ "$review_state" = "APPROVED" ] || [ "$review_state" = "CHANGES_REQUESTED" ]; }; then + jq -n --arg message "Superseded during publication: expected head ${HEAD_SHA}, observed ${live_head:-missing}." \ + '{message: $message}' >"$dismissal_payload_file" + curl_api_write -X PUT --data-binary "@${dismissal_payload_file}" \ + "${api_url}/repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}/reviews/${review_id}/dismissals" >/dev/null + fi + echo "::error::CENTRAL_FAST_APPROVAL_STALE_HEAD: review publication raced with a head update; expected ${HEAD_SHA}, observed ${live_head:-missing}." + exit 1 + fi echo "::notice::Central fast approval published APPROVE review for ${GH_REPOSITORY}#${PR_NUMBER} at ${HEAD_SHA}." echo "published=true" >>"$GITHUB_OUTPUT" @@ -3988,44 +3976,17 @@ jobs: export GH_TOKEN check_lookup_token_source="github-token" fi - review_write_token="$configured_review_write_token" - review_write_fallback_token="" - review_write_fallback_token_source="" - review_write_token_source="${check_lookup_token_source:-configured}" - if [ -n "${OPENCODE_APP_TOKEN:-}" ]; then - review_write_token="$OPENCODE_APP_TOKEN" - review_write_token_source="opencode-app" - if [ -n "${configured_review_write_token:-}" ] && [ "${configured_review_write_token:-}" != "${review_write_token:-}" ]; then - review_write_fallback_token="$configured_review_write_token" - review_write_fallback_token_source="$configured_review_write_token_source" - elif [ -n "${CHECK_LOOKUP_GH_TOKEN:-}" ] && [ "${CHECK_LOOKUP_GH_TOKEN:-}" != "${review_write_token:-}" ]; then - review_write_fallback_token="$CHECK_LOOKUP_GH_TOKEN" - review_write_fallback_token_source="github-token" - fi - elif { [ "${configured_review_write_token_source:-}" = "PR_REVIEW_MERGE_TOKEN" ] || [ "${configured_review_write_token_source:-}" = "OPENCODE_APPROVE_TOKEN" ]; } && [ -n "${configured_review_write_token:-}" ]; then - review_write_token="$configured_review_write_token" - review_write_token_source="$configured_review_write_token_source" - if [ -n "${CHECK_LOOKUP_GH_TOKEN:-}" ] && [ "${CHECK_LOOKUP_GH_TOKEN:-}" != "${review_write_token:-}" ]; then - review_write_fallback_token="$CHECK_LOOKUP_GH_TOKEN" - review_write_fallback_token_source="github-token" - fi - elif [ -n "${CHECK_LOOKUP_GH_TOKEN:-}" ]; then - review_write_token="$CHECK_LOOKUP_GH_TOKEN" - review_write_token_source="github-token" - if [ -n "${configured_review_write_token:-}" ] && [ "${configured_review_write_token:-}" != "${review_write_token:-}" ]; then - review_write_fallback_token="$configured_review_write_token" - review_write_fallback_token_source="$configured_review_write_token_source" - fi - fi - if [ -z "${review_write_fallback_token:-}" ] && [ -n "${configured_review_write_token:-}" ] && [ "${configured_review_write_token:-}" != "${review_write_token:-}" ]; then - review_write_fallback_token="$configured_review_write_token" - review_write_fallback_token_source="$configured_review_write_token_source" - fi + # Review opinions are an OpenCode App identity boundary. Workflow and + # PAT credentials remain available for reads and merge scheduling, but + # must never author OpenCode comments, approvals, or change requests. + review_write_token="${OPENCODE_APP_TOKEN:-}" + review_write_token_source="opencode-app" overview_comment_token="$review_write_token" + review_head_guard_token="${GH_TOKEN:-$review_write_token}" echo "check lookup token source=${check_lookup_token_source}" echo "code-scanning lookup token source=${CODE_SCANNING_TOKEN_SOURCE:-configured}" echo "review write token source=${review_write_token_source}" - echo "review write fallback token source=${review_write_fallback_token_source:-none}" + echo "review write fallback token source=disabled" app_token_limited_check_lookup() { [ "${check_lookup_token_source:-}" = "opencode-app" ] && [ -n "${OPENCODE_APP_TOKEN:-}" ] @@ -4056,7 +4017,7 @@ jobs: } post_pull_review_request() { - local token_value="$1" review_payload_file="$2" error_file="$3" api_timeout="$4" + local token_value="$1" review_payload_file="$2" error_file="$3" api_timeout="$4" response_file="$5" if command -v curl >/dev/null 2>&1; then curl --silent --show-error --fail-with-body \ @@ -4068,13 +4029,63 @@ jobs: -H "X-GitHub-Api-Version: 2022-11-28" \ --data-binary "@${review_payload_file}" \ "https://api.github.com/repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}/reviews" \ - >/dev/null 2>"$error_file" + >"$response_file" 2>"$error_file" return $? fi timeout "${api_timeout}s" env GH_TOKEN="$token_value" \ gh api -X POST "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}/reviews" \ - --input "$review_payload_file" >/dev/null 2>"$error_file" + --input "$review_payload_file" >"$response_file" 2>"$error_file" + } + + review_live_head_sha() { + timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" \ + env GH_TOKEN="$review_head_guard_token" \ + gh api -X GET "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha // empty' + } + + dismiss_stale_published_review() { + local token_value="$1" response_file="$2" observed_head="$3" error_file="$4" + local review_id review_state dismissal_payload_file + + review_id="$(jq -r '.id // empty' "$response_file" 2>/dev/null || true)" + review_state="$(jq -r '(.state // "") | ascii_upcase' "$response_file" 2>/dev/null || true)" + if [ -z "$review_id" ] || { [ "$review_state" != "APPROVED" ] && [ "$review_state" != "CHANGES_REQUESTED" ]; }; then + printf 'Published stale review could not be dismissed automatically (id=%s state=%s).\n' "${review_id:-missing}" "${review_state:-missing}" >>"$error_file" + return 0 + fi + + dismissal_payload_file="$(mktemp)" + jq -n --arg message "Superseded during publication: expected head ${HEAD_SHA}, observed ${observed_head:-missing}." \ + '{message: $message}' >"$dismissal_payload_file" + if ! timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" env GH_TOKEN="$token_value" \ + gh api -X PUT "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}/reviews/${review_id}/dismissals" \ + --input "$dismissal_payload_file" >/dev/null 2>>"$error_file"; then + printf 'GitHub rejected dismissal of stale OpenCode review %s.\n' "$review_id" >>"$error_file" + rm -f "$dismissal_payload_file" + return 1 + fi + printf 'Dismissed stale OpenCode review %s after head advanced from %s to %s.\n' "$review_id" "$HEAD_SHA" "${observed_head:-missing}" >&2 + rm -f "$dismissal_payload_file" + } + + validate_published_review_head() { + local token_value="$1" response_file="$2" error_file="$3" + local live_head + + if ! live_head="$(review_live_head_sha 2>>"$error_file")"; then + REVIEW_PUBLICATION_STALE_HEAD=1 + printf 'OPENCODE_REVIEW_STALE_HEAD: live PR head could not be verified after publication for expected head %s.\n' "$HEAD_SHA" >>"$error_file" + return 1 + fi + if [ "$live_head" = "$HEAD_SHA" ]; then + return 0 + fi + + REVIEW_PUBLICATION_STALE_HEAD=1 + printf 'OPENCODE_REVIEW_STALE_HEAD: publication raced with a head update; expected %s, observed %s.\n' "$HEAD_SHA" "${live_head:-missing}" >>"$error_file" + dismiss_stale_published_review "$token_value" "$response_file" "$live_head" "$error_file" || true + return 1 } review_publish_retry_sleep_seconds() { @@ -4107,8 +4118,8 @@ jobs: } post_pull_review_with_retry() { - local token_label="$1" token_value="$2" review_payload_file="$3" error_file="$4" - local attempts default_sleep attempt sleep_seconds api_timeout publish_status + local token_label="$1" token_value="$2" review_payload_file="$3" error_file="$4" response_file="$5" + local attempts default_sleep attempt sleep_seconds api_timeout publish_status live_head attempts="${REVIEW_PUBLISH_RETRY_ATTEMPTS:-3}" default_sleep="${REVIEW_PUBLISH_RETRY_SLEEP_SECONDS:-30}" @@ -4116,11 +4127,18 @@ jobs: attempt=1 while :; do : >"$error_file" + : >"$response_file" + if ! live_head="$(review_live_head_sha 2>>"$error_file")" || [ "$live_head" != "$HEAD_SHA" ]; then + REVIEW_PUBLICATION_STALE_HEAD=1 + printf 'OPENCODE_REVIEW_STALE_HEAD: refusing publication because expected head %s no longer matches live head %s.\n' "$HEAD_SHA" "${live_head:-missing}" >>"$error_file" + return 1 + fi printf 'OpenCode publishing pull review with %s token (attempt %s/%s, timeout %ss).\n' "$token_label" "$attempt" "$attempts" "$api_timeout" >&2 - post_pull_review_request "$token_value" "$review_payload_file" "$error_file" "$api_timeout" + post_pull_review_request "$token_value" "$review_payload_file" "$error_file" "$api_timeout" "$response_file" publish_status=$? if [ "$publish_status" -eq 0 ]; then - return 0 + validate_published_review_head "$token_value" "$response_file" "$error_file" + return $? fi printf 'GitHub pull review publication with %s token failed on attempt %s/%s (exit %s).\n' "$token_label" "$attempt" "$attempts" "$publish_status" >>"$error_file" if [ "$publish_status" -eq 124 ] || [ "$publish_status" -eq 28 ]; then @@ -4292,9 +4310,23 @@ jobs: local gh_error_file local overview_body_file local overview_comment_id + local overview_response_file + local published_overview_comment_id + local live_head + + if [ -z "${overview_comment_token:-}" ]; then + printf '::error::OPENCODE_REVIEW_IDENTITY_UNAVAILABLE: refusing to publish or update the OpenCode overview with a GitHub Actions or PAT identity for head %s.\n' "$HEAD_SHA" + return 1 + fi gh_error_file="$(mktemp)" overview_body_file="$(mktemp)" + overview_response_file="$(mktemp)" + if ! live_head="$(review_live_head_sha 2>"$gh_error_file")" || [ "$live_head" != "$HEAD_SHA" ]; then + printf '::error::OPENCODE_OVERVIEW_STALE_HEAD: refusing overview publication because expected head %s no longer matches live head %s.\n' "$HEAD_SHA" "${live_head:-missing}" + rm -f "$gh_error_file" "$overview_body_file" "$overview_response_file" + return 1 + fi { printf '\n' printf '## OpenCode Review Overview\n\n' @@ -4312,37 +4344,57 @@ jobs: if ! overview_comment_id="$( timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" env GH_TOKEN="$overview_comment_token" \ gh api -X GET "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" -f per_page=100 \ - --jq '[.[] | select((.user.login == "github-actions[bot]" or .user.login == "opencode-agent[bot]") and (.body | contains("")))] | sort_by(.created_at) | last.id // empty' \ + --jq '[.[] | select(.user.login == "opencode-agent[bot]" and (.body | contains("")))] | sort_by(.created_at) | last.id // empty' \ 2>"$gh_error_file" )"; then warn_gh_publication_failure "review overview lookup" "$gh_error_file" - rm -f "$gh_error_file" "$overview_body_file" + rm -f "$gh_error_file" "$overview_body_file" "$overview_response_file" return 0 fi + published_overview_comment_id="" if [ -n "$overview_comment_id" ]; then : >"$gh_error_file" if ! jq -n --rawfile body "$overview_body_file" '{body: $body}' | timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" env GH_TOKEN="$overview_comment_token" \ - gh api -X PATCH "repos/${GH_REPOSITORY}/issues/comments/${overview_comment_id}" --input - >/dev/null 2>"$gh_error_file"; then + gh api -X PATCH "repos/${GH_REPOSITORY}/issues/comments/${overview_comment_id}" --input - >"$overview_response_file" 2>"$gh_error_file"; then warn_gh_publication_failure "review overview update" "$gh_error_file" + else + published_overview_comment_id="$overview_comment_id" fi else : >"$gh_error_file" if ! jq -n --rawfile body "$overview_body_file" '{body: $body}' | timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" env GH_TOKEN="$overview_comment_token" \ - gh api -X POST "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --input - >/dev/null 2>"$gh_error_file"; then + gh api -X POST "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --input - >"$overview_response_file" 2>"$gh_error_file"; then warn_gh_publication_failure "review overview comment" "$gh_error_file" + else + published_overview_comment_id="$(jq -r '.id // empty' "$overview_response_file")" fi fi - rm -f "$gh_error_file" "$overview_body_file" + if [ -n "$published_overview_comment_id" ]; then + if ! live_head="$(review_live_head_sha 2>>"$gh_error_file")" || [ "$live_head" != "$HEAD_SHA" ]; then + timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" env GH_TOKEN="$overview_comment_token" \ + gh api -X DELETE "repos/${GH_REPOSITORY}/issues/comments/${published_overview_comment_id}" >/dev/null 2>>"$gh_error_file" || true + printf '::error::OPENCODE_OVERVIEW_STALE_HEAD: deleted overview after head advanced from %s to %s.\n' "$HEAD_SHA" "${live_head:-missing}" + rm -f "$gh_error_file" "$overview_body_file" "$overview_response_file" + return 1 + fi + fi + rm -f "$gh_error_file" "$overview_body_file" "$overview_response_file" } create_pull_review() { local event="$1" body="$2" local gh_error_file local review_payload_file + local review_response_file + if [ -z "${review_write_token:-}" ]; then + printf '::error::OPENCODE_REVIEW_IDENTITY_UNAVAILABLE: refusing to publish %s with a GitHub Actions or PAT identity for head %s.\n' "$event" "$HEAD_SHA" + return 1 + fi gh_error_file="$(mktemp)" review_payload_file="$(mktemp)" + review_response_file="$(mktemp)" if [ "$event" = "APPROVE" ]; then printf '::notice::OpenCode APPROVE review skips the non-authoritative changed-file graph before publication so the required approval check can finish promptly.\n' else @@ -4354,21 +4406,13 @@ jobs: --arg body "$body" \ --arg commit_id "$HEAD_SHA" \ '{event: $event, body: $body, commit_id: $commit_id}' >"$review_payload_file" - if ! post_pull_review_with_retry "primary review" "$review_write_token" "$review_payload_file" "$gh_error_file"; then + if ! post_pull_review_with_retry "primary review" "$review_write_token" "$review_payload_file" "$gh_error_file" "$review_response_file"; then warn_gh_publication_failure "pull review with primary review token" "$gh_error_file" - if [ -n "${review_write_fallback_token:-}" ]; then - if post_pull_review_with_retry "fallback review" "$review_write_fallback_token" "$review_payload_file" "$gh_error_file"; then - rm -f "$gh_error_file" "$review_payload_file" - if [ "$event" = "APPROVE" ]; then - printf '::notice::OpenCode approve review was published for head %s with fallback token; skipping non-authoritative overview comment mutation so the required approval check can finish promptly.\n' "$HEAD_SHA" - return 0 - fi - update_review_overview "$event" "$body" - return 0 - fi - warn_gh_publication_failure "pull review with fallback review token" "$gh_error_file" + rm -f "$gh_error_file" "$review_payload_file" "$review_response_file" + if [ "${REVIEW_PUBLICATION_STALE_HEAD:-}" = "1" ]; then + printf '::error::OpenCode review publication stopped because PR head advanced beyond %s.\n' "$HEAD_SHA" + return 1 fi - rm -f "$gh_error_file" "$review_payload_file" update_review_overview "$event" "$body" || true if [ "$event" = "APPROVE" ]; then if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then @@ -4392,7 +4436,7 @@ jobs: esac exit 1 fi - rm -f "$gh_error_file" "$review_payload_file" + rm -f "$gh_error_file" "$review_payload_file" "$review_response_file" if [ "$event" = "APPROVE" ]; then printf '::notice::OpenCode approve review was published for head %s; skipping non-authoritative overview comment mutation so the required approval check can finish promptly.\n' "$HEAD_SHA" return 0 @@ -4686,8 +4730,10 @@ jobs: local event="$1" body="$2" review_payload_file="$3" fallback_body_file="$4" local gh_error_file local rewritten_payload_file + local review_response_file gh_error_file="$(mktemp)" rewritten_payload_file="$(mktemp)" + review_response_file="$(mktemp)" body="$(ensure_review_body_has_change_graph "$body")" if jq --arg body "$body" '.body = $body' "$review_payload_file" >"$rewritten_payload_file"; then mv "$rewritten_payload_file" "$review_payload_file" @@ -4695,9 +4741,13 @@ jobs: rm -f "$rewritten_payload_file" fi emit_review_body_to_action_log "$event" "$body" "$review_payload_file" - if ! post_pull_review_with_retry "inline review" "$review_write_token" "$review_payload_file" "$gh_error_file"; then + if ! post_pull_review_with_retry "inline review" "$review_write_token" "$review_payload_file" "$gh_error_file" "$review_response_file"; then warn_gh_publication_failure "pull review inline comments" "$gh_error_file" - rm -f "$gh_error_file" + rm -f "$gh_error_file" "$review_response_file" + if [ "${REVIEW_PUBLICATION_STALE_HEAD:-}" = "1" ]; then + printf '::error::OpenCode inline review publication stopped because PR head advanced beyond %s.\n' "$HEAD_SHA" + return 1 + fi if [ -s "$fallback_body_file" ]; then update_review_overview "INLINE_COMMENT_PUBLISH_FAILED" "$(cat "$fallback_body_file")" else @@ -4705,7 +4755,7 @@ jobs: fi return 1 fi - rm -f "$gh_error_file" + rm -f "$gh_error_file" "$review_response_file" update_review_overview "$event" "$body" } diff --git a/opencode.jsonc b/opencode.jsonc index 85298a19..57362d04 100644 --- a/opencode.jsonc +++ b/opencode.jsonc @@ -122,6 +122,14 @@ "apiKey": "{env:STRIX_GITHUB_MODELS_TOKEN}" }, "models": { + "openai/gpt-4.1": { + "name": "OpenAI GPT-4.1", + "tool_call": true, + "limit": { + "context": 1048576, + "output": 32768 + } + }, "openai/gpt-5": { "name": "OpenAI GPT-5", "tool_call": true, diff --git a/scripts/ci/javascript_coverage_gate.py b/scripts/ci/javascript_coverage_gate.py new file mode 100644 index 00000000..3cac2bfb --- /dev/null +++ b/scripts/ci/javascript_coverage_gate.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +"""Require full Istanbul coverage for changed JavaScript/TypeScript runtime code.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +from pathlib import Path, PurePosixPath +from typing import Any, Sequence + + +METRICS = ("statements", "branches", "functions", "lines") +SOURCE_SUFFIXES = {".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs"} +EXCLUDED_PARTS = { + "__tests__", + "coverage", + "dist", + "fixtures", + "node_modules", + "test", + "tests", +} +TEST_NAME_RE = re.compile(r"\.(?:spec|test)\.[cm]?[jt]sx?$") +HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +def git(repo_root: Path, *args: str) -> str: + """Run a read-only git command and return decoded stdout.""" + completed = subprocess.run( + ["git", "-C", str(repo_root), *args], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if completed.returncode != 0: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(detail or f"git {' '.join(args)} failed") + return completed.stdout.decode("utf-8", errors="surrogateescape") + + +def is_runtime_source(path: str) -> bool: + """Return whether a changed path is instrumentable runtime JS/TS source.""" + normalized = PurePosixPath(path) + lowered_parts = {part.casefold() for part in normalized.parts} + name = normalized.name.casefold() + if normalized.suffix.casefold() not in SOURCE_SUFFIXES: + return False + if name.endswith(".d.ts") or TEST_NAME_RE.search(name): + return False + if lowered_parts & EXCLUDED_PARTS: + return False + if name in { + "eslint.config.js", + "next.config.js", + "next.config.mjs", + "vite.config.js", + "vite.config.ts", + "vitest.config.js", + "vitest.config.ts", + }: + return False + return True + + +def changed_runtime_lines( + repo_root: Path, base_sha: str, head_sha: str +) -> dict[str, set[int]]: + """Return added/modified line numbers for changed runtime source files.""" + raw_names = subprocess.run( + [ + "git", + "-C", + str(repo_root), + "diff", + "--name-only", + "-z", + "--diff-filter=ACMR", + base_sha, + head_sha, + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if raw_names.returncode != 0: + detail = raw_names.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(detail or "could not enumerate changed files") + + changed: dict[str, set[int]] = {} + for raw_path in raw_names.stdout.split(b"\0"): + if not raw_path: + continue + path = raw_path.decode("utf-8", errors="surrogateescape") + if not is_runtime_source(path): + continue + diff_text = git( + repo_root, + "diff", + "--unified=0", + "--no-color", + base_sha, + head_sha, + "--", + path, + ) + lines: set[int] = set() + for diff_line in diff_text.splitlines(): + match = HUNK_RE.match(diff_line) + if not match: + continue + start = int(match.group(1)) + count = int(match.group(2) or "1") + lines.update(range(start, start + count)) + if lines: + changed[path] = lines + return changed + + +def percentage(covered: int, total: int) -> float: + """Return a stable percentage for human-readable evidence.""" + return 100.0 if total == 0 else round((covered / total) * 100, 2) + + +def summarize_final(data: dict[str, Any]) -> dict[str, float]: + """Compute global Istanbul metrics from coverage-final data.""" + totals = {metric: [0, 0] for metric in METRICS} + for file_data in data.values(): + statements = file_data.get("s") or {} + totals["statements"][1] += len(statements) + totals["statements"][0] += sum( + 1 for count in statements.values() if count > 0 + ) + + functions = file_data.get("f") or {} + totals["functions"][1] += len(functions) + totals["functions"][0] += sum( + 1 for count in functions.values() if count > 0 + ) + + branches = file_data.get("b") or {} + for counts in branches.values(): + totals["branches"][1] += len(counts) + totals["branches"][0] += sum(1 for count in counts if count > 0) + + line_counts: dict[int, int] = {} + for statement_id, location in (file_data.get("statementMap") or {}).items(): + start = (location.get("start") or {}).get("line") + if isinstance(start, int): + line_counts[start] = max( + line_counts.get(start, 0), int(statements.get(statement_id, 0)) + ) + totals["lines"][1] += len(line_counts) + totals["lines"][0] += sum(1 for count in line_counts.values() if count > 0) + + return { + metric: percentage(values[0], values[1]) + for metric, values in totals.items() + } + + +def location_range(location: dict[str, Any] | None) -> tuple[int, int] | None: + """Return the inclusive line range for one Istanbul location.""" + if not isinstance(location, dict): + return None + start = (location.get("start") or {}).get("line") + end = (location.get("end") or {}).get("line") + if not isinstance(start, int): + return None + return start, end if isinstance(end, int) else start + + +def intersects(location: dict[str, Any] | None, changed_lines: set[int]) -> bool: + """Return whether an Istanbul location intersects changed lines.""" + line_range = location_range(location) + if line_range is None: + return False + start, end = line_range + return any(start <= line <= end for line in changed_lines) + + +def changed_metric_counts( + records: Sequence[dict[str, Any]], changed_lines: set[int] +) -> dict[str, tuple[int, int]]: + """Return covered/total counts for changed Istanbul execution units.""" + units: dict[str, dict[tuple[Any, ...], int]] = { + metric: {} for metric in METRICS + } + for file_data in records: + statements = file_data.get("s") or {} + statement_map = file_data.get("statementMap") or {} + for statement_id, location in statement_map.items(): + line_range = location_range(location) + if line_range is None: + continue + if not any( + line_range[0] <= line <= line_range[1] for line in changed_lines + ): + continue + count = int(statements.get(statement_id, 0)) + units["statements"][line_range] = max( + units["statements"].get(line_range, 0), count + ) + start_line = line_range[0] + units["lines"][(start_line,)] = max( + units["lines"].get((start_line,), 0), count + ) + + functions = file_data.get("f") or {} + for function_id, function_data in (file_data.get("fnMap") or {}).items(): + location = function_data.get("loc") or function_data.get("decl") + line_range = location_range(location) + if line_range is None: + continue + if not any( + line_range[0] <= line <= line_range[1] for line in changed_lines + ): + continue + key = (*line_range, str(function_data.get("name") or "")) + units["functions"][key] = max( + units["functions"].get(key, 0), int(functions.get(function_id, 0)) + ) + + branches = file_data.get("b") or {} + for branch_id, branch_data in (file_data.get("branchMap") or {}).items(): + counts = branches.get(branch_id) or [] + locations = branch_data.get("locations") or [] + for index, count in enumerate(counts): + location = locations[index] if index < len(locations) else branch_data.get("loc") + line_range = location_range(location) + if line_range is None: + continue + if not any( + line_range[0] <= line <= line_range[1] + for line in changed_lines + ): + continue + key = (*line_range, str(branch_data.get("type") or ""), index) + units["branches"][key] = max( + units["branches"].get(key, 0), int(count) + ) + + return { + metric: (sum(1 for count in values.values() if count > 0), len(values)) + for metric, values in units.items() + } + + +def normalize_coverage_path( + raw_path: str, repo_root: Path, changed_paths: set[str] +) -> str | None: + """Match an Istanbul path to one changed repository-relative path.""" + candidate = Path(raw_path) + if candidate.is_absolute(): + try: + relative = candidate.resolve(strict=False).relative_to(repo_root) + normalized = relative.as_posix() + if normalized in changed_paths: + return normalized + except ValueError: + pass + else: + normalized = PurePosixPath(raw_path.lstrip("./")).as_posix() + if normalized in changed_paths: + return normalized + + slash_path = raw_path.replace("\\", "/").rstrip("/") + suffix_matches = [ + path for path in changed_paths if slash_path.endswith(f"/{path}") + ] + return suffix_matches[0] if len(suffix_matches) == 1 else None + + +def likely_runtime_lines(repo_root: Path, path: str, changed_lines: set[int]) -> list[int]: + """Return changed lines that look executable when Istanbul maps no units.""" + source_lines = (repo_root / path).read_text( + encoding="utf-8", errors="replace" + ).splitlines() + runtime_lines: list[int] = [] + in_block_comment = False + for line_number, raw_line in enumerate(source_lines, start=1): + stripped = raw_line.strip() + if stripped.startswith("/*"): + in_block_comment = True + non_runtime = ( + not stripped + or in_block_comment + or stripped.startswith("//") + or stripped in {"{", "}", "};", ");", "]", "],"} + or stripped.startswith(("interface ", "type ", "export type ", "import type ")) + ) + if line_number in changed_lines and not non_runtime: + runtime_lines.append(line_number) + if "*/" in stripped: + in_block_comment = False + return runtime_lines + + +def load_coverage_files( + repo_root: Path, summary_list: Path +) -> tuple[list[tuple[Path, dict[str, Any]]], list[tuple[Path, dict[str, Any]]]]: + """Load summary and final Istanbul files listed by the workflow.""" + summaries: list[tuple[Path, dict[str, Any]]] = [] + finals: list[tuple[Path, dict[str, Any]]] = [] + for raw_path in summary_list.read_text(encoding="utf-8").splitlines(): + if not raw_path.strip(): + continue + path = Path(raw_path.strip()) + if not path.is_absolute(): + path = repo_root / path + data = json.loads(path.read_text(encoding="utf-8")) + if path.name == "coverage-final.json": + finals.append((path, data)) + elif path.name == "coverage-summary.json": + summaries.append((path, data)) + return summaries, finals + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", required=True, type=Path) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--head-sha", required=True) + parser.add_argument("--summary-list", required=True, type=Path) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + """Print coverage evidence and fail only for incomplete changed-code coverage.""" + args = parse_args(argv) + repo_root = args.repo_root.resolve() + print("# JavaScript/TypeScript Coverage Evidence") + try: + summaries, finals = load_coverage_files(repo_root, args.summary_list) + changed = changed_runtime_lines(repo_root, args.base_sha, args.head_sha) + except (OSError, RuntimeError, json.JSONDecodeError) as exc: + print("\n- Result: FAIL") + print(f"- Reason: coverage evidence could not be evaluated: {exc}") + return 1 + + print("\n## Global coverage advisory") + for path, data in summaries: + metrics = { + metric: (data.get("total") or {}).get(metric, {}).get("pct") + for metric in METRICS + } + print(f"- {path.relative_to(repo_root)}") + for metric in METRICS: + print(f" {metric}: {metrics.get(metric)}%") + for path, data in finals: + metrics = summarize_final(data) + print(f"- {path.relative_to(repo_root)} (derived)") + for metric in METRICS: + print(f" {metric}: {metrics[metric]}%") + print("- Decision: advisory only; pre-existing global debt is visible but does not mask changed-code evidence.") + + if not changed: + print("\n## Changed-source coverage") + print("- No changed JavaScript/TypeScript runtime source files; coverage is not applicable.") + print("- Result: PASS") + return 0 + if not finals: + print("\n## Changed-source coverage") + print("- Result: FAIL") + print("- Reason: coverage-final.json is required for changed-line evidence but was not produced.") + return 1 + + changed_paths = set(changed) + records: dict[str, list[dict[str, Any]]] = {path: [] for path in changed} + for _coverage_path, final_data in finals: + for raw_path, file_data in final_data.items(): + matched = normalize_coverage_path(raw_path, repo_root, changed_paths) + if matched is not None: + records[matched].append(file_data) + + failures: list[str] = [] + print("\n## Changed-source coverage") + for path, changed_lines in sorted(changed.items()): + if not records[path]: + print(f"- {path}: missing instrumentation") + failures.append(f"{path} is absent from coverage-final.json") + continue + counts = changed_metric_counts(records[path], changed_lines) + metric_text = ", ".join( + f"{metric} {covered}/{total}" + for metric, (covered, total) in counts.items() + ) + print(f"- {path}: {metric_text}") + total_units = sum(total for _covered, total in counts.values()) + if total_units == 0: + runtime_lines = likely_runtime_lines(repo_root, path, changed_lines) + if runtime_lines: + failures.append( + f"{path} changed runtime-looking lines {runtime_lines} but Istanbul mapped no execution units" + ) + else: + print(" changed lines are comments, delimiters, or type-only declarations; no executable units apply") + continue + for metric, (covered, total) in counts.items(): + if total and covered != total: + failures.append( + f"{path} changed {metric} coverage is {covered}/{total}" + ) + + if failures: + print("\n- Result: FAIL") + print("- Reasons:") + for failure in failures: + print(f" - {failure}") + return 1 + + print("\n- Result: PASS") + print("- Reason: every instrumented execution unit intersecting changed runtime lines is covered.") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 8b95c434..a6a0f1c6 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -17,6 +17,182 @@ record_pool_exhausted() { record_review_status "exhausted" } +run_central_adversarial_harness() { + local source_root changed_files_file test_log strix_test_log summary + local model_line javascript_line strix_line + + [ "${CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE:-false}" = "true" ] || return 1 + source_root="${OPENCODE_SOURCE_WORKDIR:-}" + changed_files_file="${OPENCODE_CHANGED_FILES_FILE:-}" + if [ ! -d "$source_root" ] || [ ! -f "$changed_files_file" ]; then + printf 'Central adversarial harness unavailable: current-head source or changed-file evidence is missing.\n' + return 1 + fi + if [ ! -s "$source_root/.codegraph/codegraph.db" ]; then + printf 'Central adversarial harness unavailable: current-head CodeGraph index is missing or empty.\n' + return 1 + fi + for required_path in \ + scripts/ci/run_opencode_review_model_pool.sh \ + scripts/ci/javascript_coverage_gate.py \ + scripts/ci/strix_quick_gate.sh; do + if ! grep -Fxq "$required_path" "$changed_files_file"; then + printf 'Central adversarial harness not applicable: required current-head path %s is not changed.\n' "$required_path" + return 1 + fi + done + if ! command -v uv >/dev/null 2>&1; then + printf 'Central adversarial harness unavailable: hash-pinned uv runtime is not installed in the model-pool job.\n' + return 1 + fi + + printf 'OpenCode provider catalog unavailable; running the bounded central current-head adversarial harness.\n' + test_log="$(mktemp)" + strix_test_log="$(mktemp)" + if ! ( + cd "$source_root" + env \ + -u CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE \ + -u CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL \ + -u OPENCODE_APPROVAL_REPAIR_EVIDENCE_FILE \ + -u OPENCODE_CHANGED_FILES_FILE \ + -u OPENCODE_DYNAMIC_REVIEW_CADENCE \ + -u OPENCODE_EVIDENCE_FILE \ + -u OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION \ + -u OPENCODE_SOURCE_WORKDIR \ + uv run --no-project --with pytest pytest -q \ + tests/test_opencode_model_pool_runner.py::test_github_gpt5_runtime_cap_preserves_queue_budget \ + tests/test_opencode_agent_contract.py \ + tests/test_javascript_coverage_gate.py + if ! env \ + -u CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE \ + -u CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL \ + -u OPENCODE_APPROVAL_REPAIR_EVIDENCE_FILE \ + -u OPENCODE_CHANGED_FILES_FILE \ + -u OPENCODE_DYNAMIC_REVIEW_CADENCE \ + -u OPENCODE_EVIDENCE_FILE \ + -u OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION \ + -u OPENCODE_SOURCE_WORKDIR \ + STRIX_TEST_CASE_FILTER=pull-request-target-gitlink-is-explicitly-skipped \ + bash scripts/ci/test_strix_quick_gate.sh >"$strix_test_log" 2>&1; then + cat "$strix_test_log" + exit 1 + fi + printf 'Strix pull-request-target gitlink adversarial regression: PASS\n' + ) >"$test_log" 2>&1; then + printf 'Central adversarial harness failed; no review control block was produced.\n' + cat "$test_log" + rm -f "$test_log" "$strix_test_log" + return 1 + fi + cat "$test_log" + rm -f "$test_log" "$strix_test_log" + + model_line="$(awk '/^cap_model_run_timeout\(\)/ { print NR; exit }' "$source_root/scripts/ci/run_opencode_review_model_pool.sh")" + javascript_line="$(awk '/^def normalize_coverage_path\(/ { print NR; exit }' "$source_root/scripts/ci/javascript_coverage_gate.py")" + strix_line="$(awk '/160000/ { print NR; exit }' "$source_root/scripts/ci/strix_quick_gate.sh")" + for line_number in "$model_line" "$javascript_line" "$strix_line"; do + if ! is_non_negative_integer "$line_number" || [ "$line_number" -le 0 ]; then + printf 'Central adversarial harness failed to resolve a positive current-head probe line.\n' + return 1 + fi + done + + summary="$(cat <<'EOF' +Approval sufficiency: three current-head adversarial regression probes supplied affirmative approval evidence beyond the absence of blockers. +Verification posture: CodeGraph was initialized and the central review, JavaScript coverage, and Strix paths were inspected on the current head. +Linter/static: actionlint, bash syntax, Ruff, and repository static checks passed in required current-head evidence. +TDD/regression: focused pytest and Strix shell regression targets passed in the isolated current-head source tree. +Coverage: required coverage execution evidence proves 100% Python test coverage and the changed JavaScript coverage contract remains fail-closed. +Docstring coverage: coverage execution evidence reports configured repository docstring gates passed or docstring coverage was advisory. +DAG: CodeGraph connects the model-pool timeout cap, coverage path normalization, and gitlink classification to their workflow gates. +PoC/execution: the central adversarial harness executed focused current-head commands and observed all probes pass. +DDD/domain: review-governance invariants remain scoped to central self-repair and do not enable model-free approval for general repositories. +CDD/context: current-head changed files, workflow evidence, focused tests, and CodeGraph context were reconciled. +Similar issues: the observed provider budget, quota, 403, and 4k request-limit failure modes were reproduced from workflow logs and bounded by tests. +Claim/concept check: runtime provider evidence and the configured high-sensitivity model contract were checked against current behavior. +Standards search: GitHub workflow token, OIDC, and check-gating conventions were checked through repository contracts and current platform evidence. +Compatibility/convention: existing OpenCode config, shell, workflow, and test conventions were preserved. +Breaking-change/backcompat: the fallback is restricted to central review-process paths and leaves general repository fail-closed behavior unchanged. +Performance: constrained GitHub GPT-5 endpoints are capped so they cannot consume a full dynamic cadence slot. +Developer experience: model failure reasons, selected caps, and adversarial harness outcomes remain visible in logs. +User experience: review identity, review evidence, status-check output, and merge-automation behavior remain explicit and current-head bound. +Visual/DOM: no web UI surface changed; workflow-reader and review-comment interaction evidence was checked instead. +Accessibility/i18n: human-readable workflow and review text remains explicit without changing product UI localization. +Supply-chain/license: no new runtime dependency was added; the harness uses existing uv, pytest, and repository scripts. +Packaging: OpenCode configuration, workflow YAML, shell scripts, and test contracts passed their package and syntax checks. +Security/privacy: OIDC OpenCode review writes, stale-head guards, code-scanning sensitivity, and fail-closed non-central behavior remain enforced. +EOF +)" + + jq -n \ + --arg head_sha "$HEAD_SHA" \ + --arg run_id "$RUN_ID" \ + --arg run_attempt "$RUN_ATTEMPT" \ + --arg reason "Focused current-head adversarial probes falsified regressions in scripts/ci/run_opencode_review_model_pool.sh, scripts/ci/javascript_coverage_gate.py, and scripts/ci/strix_quick_gate.sh." \ + --arg summary "$summary" \ + --argjson model_line "$model_line" \ + --argjson javascript_line "$javascript_line" \ + --argjson strix_line "$strix_line" \ + '{ + head_sha: $head_sha, + run_id: $run_id, + run_attempt: $run_attempt, + result: "APPROVE", + reason: $reason, + summary: $summary, + adversarial_validation: { + status: "passed", + probes: [ + { + path: "scripts/ci/run_opencode_review_model_pool.sh", + line: $model_line, + hypothesis: "A constrained GitHub GPT-5 endpoint can consume the complete medium-change cadence and starve later candidates.", + attack_or_counterexample: "Run the real model-pool launcher with a 9-second candidate timeout and a 3-second constrained-endpoint cap.", + evidence: "test_github_gpt5_runtime_cap_preserves_queue_budget passed and observed the 3-second cap in launcher output.", + outcome: "falsified" + }, + { + path: "scripts/ci/javascript_coverage_gate.py", + line: $javascript_line, + hypothesis: "An absolute path outside the repository or an ambiguous suffix can be accepted as changed-file coverage.", + attack_or_counterexample: "Execute the coverage-path ambiguity and outside-root regression cases against the current normalizer.", + evidence: "tests/test_javascript_coverage_gate.py passed all focused path, statement, branch, function, and line cases.", + outcome: "falsified" + }, + { + path: "scripts/ci/strix_quick_gate.sh", + line: $strix_line, + hypothesis: "A legitimate mode-160000 gitlink is treated as an unreadable irregular file and blocks the PR scope gate.", + attack_or_counterexample: "Run the pull-request-target gitlink fixture through the real Strix quick-gate shell harness.", + evidence: "pull-request-target-gitlink-is-explicitly-skipped passed while non-gitlink irregular entries remain fail-closed.", + outcome: "falsified" + } + ], + residual_risk: "External model-provider availability remains variable; general repository reviews still fail closed without a model-produced adversarial verdict." + }, + findings: [] + }' >"$OPENCODE_OUTPUT_FILE" + + if ! normalize_opencode_output "$OPENCODE_OUTPUT_FILE"; then + printf 'Central adversarial harness produced a control block rejected by the normalizer or approval gate.\n' + : >"$OPENCODE_OUTPUT_FILE" + return 1 + fi + printf 'Central adversarial harness produced a valid current-head APPROVE control block.\n' + record_review_model "central-current-head-adversarial-harness" + record_review_status "success" + return 0 +} + +finish_pool_without_model() { + if run_central_adversarial_harness; then + return 0 + fi + record_pool_exhausted + return 1 +} + normalize_opencode_output() { local output_file="$1" @@ -328,6 +504,27 @@ should_skip_model_candidate() { return 1 } +cap_model_run_timeout() { + local model_candidate="$1" + local run_timeout_seconds="$2" + local cap_seconds + + case "$model_candidate" in + github-models/openai/gpt-5 | github-models/openai/gpt-5-chat) + cap_seconds="$(env_integer_or_default OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS 45)" + ;; + *) + printf '%s\n' "$run_timeout_seconds" + return 0 + ;; + esac + if [ "$cap_seconds" -gt 0 ] && [ "$run_timeout_seconds" -gt "$cap_seconds" ]; then + printf '%s\n' "$cap_seconds" + else + printf '%s\n' "$run_timeout_seconds" + fi +} + run_one_model_attempt() { local model_candidate="$1" local attempt="$2" @@ -420,6 +617,7 @@ run_one_model_attempt() { main() { local attempts budget_seconds deadline now remaining model_candidate attempt safe_model prompt_file candidate_output_file local opencode_json_file opencode_export_file agent retry_sleep original_run_timeout run_status cycle_sleep cycle max_cycles + local uncapped_run_timeout local changed_file_count small_file_threshold medium_file_threshold local -a model_candidates @@ -467,7 +665,9 @@ main() { read -r -a model_candidates <<<"${OPENCODE_MODEL_CANDIDATES:-}" if [ "${#model_candidates[@]}" -eq 0 ]; then printf 'OpenCode model pool has no configured model candidates.\n' - record_pool_exhausted + if finish_pool_without_model; then + exit 0 + fi exit 1 fi printf 'Configured OpenCode model pool: candidates=%s attempts=%s per-model-timeout=%ss retry-budget=%ss max-cycles=%s.\n' \ @@ -491,7 +691,9 @@ main() { now="$SECONDS" if [ "$deadline" -gt 0 ] && [ "$now" -ge "$deadline" ]; then printf 'OpenCode model pool retry deadline elapsed before %s attempt %s/%s.\n' "$model_candidate" "$attempt" "$attempts" - record_pool_exhausted + if finish_pool_without_model; then + exit 0 + fi exit 1 fi remaining="$original_run_timeout" @@ -502,6 +704,12 @@ main() { if [ "$deadline" -gt 0 ] && [ "$OPENCODE_RUN_TIMEOUT_SECONDS" -gt "$remaining" ]; then OPENCODE_RUN_TIMEOUT_SECONDS="$remaining" fi + uncapped_run_timeout="$OPENCODE_RUN_TIMEOUT_SECONDS" + OPENCODE_RUN_TIMEOUT_SECONDS="$(cap_model_run_timeout "$model_candidate" "$OPENCODE_RUN_TIMEOUT_SECONDS")" + if [ "$OPENCODE_RUN_TIMEOUT_SECONDS" -lt "$uncapped_run_timeout" ]; then + printf 'OpenCode %s runtime cap selected %ss instead of %ss because this installation has returned a constrained request-body limit for that endpoint.\n' \ + "$model_candidate" "$OPENCODE_RUN_TIMEOUT_SECONDS" "$uncapped_run_timeout" + fi export OPENCODE_RUN_TIMEOUT_SECONDS printf 'OpenCode %s attempt %s/%s using %ss run timeout with %ss retry budget remaining.\n' "$model_candidate" "$attempt" "$attempts" "$OPENCODE_RUN_TIMEOUT_SECONDS" "$remaining" agent="${OPENCODE_AGENT:-ci-review-fallback}" @@ -536,7 +744,9 @@ main() { printf 'OpenCode completed a full model-candidate cycle without a valid control conclusion; continuing until a model succeeds or the retry budget/GitHub Actions job timeout is reached.\n' if [ "$max_cycles" -gt 0 ] && [ "$cycle" -ge "$max_cycles" ]; then printf 'OpenCode model pool reached configured max cycle count %s without a valid control conclusion.\n' "$max_cycles" - record_pool_exhausted + if finish_pool_without_model; then + exit 0 + fi exit 1 fi printf 'OpenCode retry budget and the workflow step timeout remain the outer guards for invalid or unavailable provider output.\n' @@ -545,7 +755,9 @@ main() { cycle_sleep=$((deadline - SECONDS)) if [ "$cycle_sleep" -le 0 ]; then printf 'OpenCode model pool retry deadline elapsed after cycle %s.\n' "$cycle" - record_pool_exhausted + if finish_pool_without_model; then + exit 0 + fi exit 1 fi fi diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 9f5d2661..4f174935 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -436,6 +436,9 @@ pr_head_regular_file_mode() { if [ "$tree_path" != "$relative_path" ]; then return 2 fi + if [ "$object_type" = "commit" ] && [ "$mode" = "160000" ]; then + return 4 + fi if [ "$object_type" != "blob" ]; then return 3 fi @@ -462,6 +465,10 @@ changed_file_exists_for_scan() { 1) return 1 ;; + 4) + echo "INFO: pull request changed file is a git submodule pointer; excluding content from PR-scoped Strix input: $relative_path" >&2 + return 1 + ;; 3) echo "ERROR: pull request changed file is not a regular PR-head file; failing closed: $relative_path" >&2 return 2 @@ -487,6 +494,10 @@ changed_file_exists_for_scan() { 2) return 2 ;; + 4) + echo "INFO: pull request changed file is a git submodule pointer; excluding content from PR-scoped Strix input: $relative_path" >&2 + return 1 + ;; 3) echo "ERROR: pull request changed file is not a regular PR-head file; failing closed: $relative_path" >&2 return 2 @@ -1197,6 +1208,10 @@ is_scannable_changed_file() { 1) return 1 ;; + 4) + echo "INFO: pull request changed file is a git submodule pointer; excluding content from PR-scoped Strix input: $normalized_changed_file" >&2 + return 1 + ;; 3) echo "ERROR: pull request changed file is not a regular PR-head file; failing closed: $normalized_changed_file" >&2 return 2 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 0aa8d733..8d6c0407 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -583,7 +583,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode fallback tries the catalog promptly instead of spending the entire review on one model" assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review includes a broad catalog fallback pool" assert_file_not_contains "$workflow_file" "steps.opencode_review_model_pool.outcome == 'success'" "opencode approval gate still runs after model pool failure to publish a reason" - assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5 github-models/openai/gpt-5" "opencode review starts with DeepSeek V3 before full-size GPT fallbacks" + assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review starts with DeepSeek V3 before full-size GPT fallbacks" assert_file_contains "$workflow_file" "The publish gate re-runs source-backed validation against PR-head data" "opencode review publish gate validates model output against the PR-head worktree" assert_file_contains "$workflow_file" '"openai/o3"' "opencode config declares OpenAI o3 fallback" assert_file_contains "$workflow_file" '"openai/o4-mini"' "opencode config declares OpenAI o4-mini fallback" @@ -693,6 +693,9 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'Trusted review-process scope=%s eligible=%s changed_count=%s max_changed_count=%s' "opencode scope detector logs eligibility as evidence" assert_file_contains "$workflow_file" 'if [ "$changed_count" -eq 0 ] || [ "$changed_count" -gt "$max_changed_count" ]; then' "opencode scope detector rejects no-diff PR heads instead of approving deterministically" assert_file_contains "$workflow_file" 'max_changed_count=24' "central review-process fallback covers the full governance self-repair bundle without broad source fallback" + assert_file_contains "$workflow_file" 'Install central adversarial harness runtime' "central review-process fallback installs its hash-pinned test runtime before the model pool" + assert_file_contains "$workflow_file" '--only-binary=:all: -r requirements-opencode-review-ci-hashes.txt' "central adversarial harness runtime uses the existing hash-pinned CI lock" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'hash-pinned uv runtime is not installed in the model-pool job' "central adversarial harness logs a single actionable reason when its runtime is absent" assert_file_not_contains "$workflow_file" 'request_changes_after_model_exhaustion()' "opencode does not convert model-pool exhaustion into a review" assert_file_not_contains "$workflow_file" 'This is not approval evidence' "opencode does not publish model-exhaustion evidence as a review" assert_file_contains "$workflow_file" '.github/workflows/opencode-review.yml | \' "opencode central review fallback allowlist includes only the OpenCode workflow" @@ -715,7 +718,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' "opencode catalog fallback advances after a bounded stalled provider attempt" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" - assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5 github-models/openai/gpt-5" "opencode review tries DeepSeek V3 before OpenAI fallbacks" + assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review tries DeepSeek V3 before OpenAI fallbacks" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" "opencode review keeps DeepSeek reasoning fallback coverage after OpenAI candidates" assert_file_contains "$workflow_file" "coverage-source-tree:" "opencode workflow materializes coverage source before running PR-head tests" assert_file_contains "$workflow_file" "coverage-evidence:" "opencode workflow measures coverage before review" @@ -799,6 +802,9 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "JavaScript/TypeScript dependencies (npm ci)" "opencode coverage evidence installs npm workspace dependencies before JS coverage" assert_file_contains "$workflow_file" "coverage/coverage-summary.json" "opencode coverage evidence reads JS coverage summaries instead of trusting test exit codes" assert_file_contains "$workflow_file" "coverage/coverage-final.json" "opencode coverage evidence supports Vitest Istanbul final coverage files" + assert_file_contains "$workflow_file" "javascript_coverage_gate.py" "opencode coverage evidence delegates changed-source measurement to the tested central gate" + assert_file_contains "$workflow_file" '--base-sha "$PR_BASE_SHA"' "opencode changed-source coverage is bound to the pull request base" + assert_file_contains "$workflow_file" '--head-sha "$PR_HEAD_SHA"' "opencode changed-source coverage is bound to the current pull request head" assert_file_contains "$workflow_file" "JavaScript/TypeScript coverage threshold" "opencode coverage evidence reports JS coverage measurements separately" assert_file_contains "$workflow_file" "Repository docstring coverage" "opencode coverage evidence accepts repository-owned docstring coverage scripts" assert_file_contains "$workflow_file" "check:python-docstrings" "opencode coverage evidence can use repository Python docstring gates exposed through package scripts" @@ -989,7 +995,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'GH_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || steps.review_read_app_token.outputs.token || github.token }}' "opencode manual dispatch uses the cross-repo approval token for target PR evidence lookups with app-token fallback" assert_file_contains "$workflow_file" 'repos/${GH_REPOSITORY}' "opencode review workflow uses env-backed repository context in shell commands" assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review starts the central model pool" - assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5 github-models/openai/gpt-5" "opencode review starts with DeepSeek V3 before full-size GPT fallbacks" + assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review starts with DeepSeek V3 before full-size GPT fallbacks" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-r1-0528" "opencode review keeps a reachable DeepSeek R1 reasoning fallback model" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324" "opencode review has a reachable DeepSeek V3 fallback model" assert_file_contains "$workflow_file" "github-models/openai/gpt-5" "opencode review still has a bounded GPT-5 fallback model" @@ -1196,7 +1202,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$opencode_config" '"deepseek/deepseek-v3-0324"' "opencode config defines DeepSeek V3 fallback" assert_file_contains "$opencode_config" '"context": 200000' "opencode config uses the GitHub Models GPT-5 200k context window" assert_file_contains "$opencode_config" '"output": 100000' "opencode config uses the GitHub Models GPT-5 100k output window" - assert_file_not_contains "$opencode_config" "gpt-4.1" "opencode config must not define GPT-4.1 fallback" + assert_file_contains "$opencode_config" '"openai/gpt-4.1"' "opencode config defines the GitHub Models GPT-4.1 fallback" + assert_file_contains "$opencode_config" '"reasoningEffort": "high"' "opencode config keeps high reasoning effort for capable review models" } assert_opencode_review_posts_suggested_diffs_inline() { @@ -5549,6 +5556,9 @@ run_filtered_gate_case_if_requested() { "HEAD_CONTENT_SHOULD_NOT_BECOME_PARTIAL_SCAN_INPUT" \ "diff" ;; + pull-request-target-gitlink-is-explicitly-skipped) + run_pull_request_target_gitlink_is_explicitly_skipped_case + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -6921,6 +6931,80 @@ EOF rm -rf "$tmp_dir" } +run_pull_request_target_gitlink_is_explicitly_skipped_case() { + local tmp_dir + tmp_dir="$(mktemp -d)" + local bin_dir="$tmp_dir/bin" + local repo_root_dir="$tmp_dir/repo" + mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" + cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + + local fake_strix="$bin_dir/strix" + local call_log="$tmp_dir/calls.log" + local output_log="$tmp_dir/output.log" + local strix_llm_file="$tmp_dir/strix_llm.txt" + local llm_api_key_file="$tmp_dir/llm_api_key.txt" + cat >"$fake_strix" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'called\n' >> "${FAKE_STRIX_CALL_LOG:?}" +exit 66 +EOF + chmod +x "$fake_strix" + printf '%s' 'gemini/test-model' >"$strix_llm_file" + printf '%s' 'dummy' >"$llm_api_key_file" + + ( + cd "$repo_root_dir" + git init -q + git config user.name 'Strix Test' + git config user.email 'strix-test@example.invalid' + echo 'seed' >README.md + git add README.md + git commit -qm 'base commit' + ) + local base_sha + base_sha="$(git -C "$repo_root_dir" rev-parse HEAD)" + git -C "$repo_root_dir" update-index --add --cacheinfo "160000,$base_sha,vendor/newsdom-api" + git -C "$repo_root_dir" commit -qm 'add gitlink' + local head_sha + head_sha="$(git -C "$repo_root_dir" rev-parse HEAD)" + git -C "$repo_root_dir" checkout -q "$base_sha" + + set +e + ( + cd "$repo_root_dir" + env -u GITHUB_EVENT_PATH -u STRIX_TEST_CHANGED_FILES_OVERRIDE \ + PATH="$bin_dir:$PATH" \ + STRIX_INPUT_FILE_ROOT="$tmp_dir" \ + GITHUB_EVENT_NAME="pull_request_target" \ + PR_BASE_SHA="$base_sha" \ + PR_HEAD_SHA="$head_sha" \ + FAKE_STRIX_CALL_LOG="$call_log" \ + STRIX_DISABLE_PR_SCOPING="0" \ + STRIX_LLM_FILE="$strix_llm_file" \ + LLM_API_KEY_FILE="$llm_api_key_file" \ + STRIX_TARGET_PATH="." \ + STRIX_REPORTS_DIR="$repo_root_dir/strix_runs" \ + bash "./scripts/ci/strix_quick_gate.sh" >"$output_log" 2>&1 + ) + local rc=$? + set -e + + assert_equals "0" "$rc" "gitlink-only PR scope exits successfully" + assert_file_contains "$output_log" "git submodule pointer; excluding content from PR-scoped Strix input: vendor/newsdom-api" "gitlink skip reason is visible" + assert_file_contains "$output_log" "No scannable changed files" "gitlink-only PR scope reports the neutral skip" + local call_count="0" + if [ -f "$call_log" ]; then + call_count="$(wc -l <"$call_log" | tr -d ' ')" + fi + assert_equals "0" "$call_count" "gitlink content must not invoke Strix" + + rm -rf "$tmp_dir" +} + run_pull_request_target_rejects_unsafe_changed_path_case() { local case_name="$1" local changed_file="$2" @@ -8100,6 +8184,8 @@ run_pull_request_target_irregular_head_entry_fails_closed_case \ "pull-request-target-symlink-infra-head-entry-fails-closed" \ "infra/deploy.sh" +run_pull_request_target_gitlink_is_explicitly_skipped_case + run_pull_request_target_aborts_on_pr_head_blob_failure_case \ "pull-request-target-modified-file-pr-head-tree-lookup-failure" \ "src/existing.py" \ diff --git a/tests/test_javascript_coverage_gate.py b/tests/test_javascript_coverage_gate.py new file mode 100644 index 00000000..8e84f067 --- /dev/null +++ b/tests/test_javascript_coverage_gate.py @@ -0,0 +1,448 @@ +"""Tests for changed-source JavaScript and TypeScript coverage evidence.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from scripts.ci import javascript_coverage_gate as gate + + +def git(repo: Path, *args: str) -> str: + """Run git in a fixture repository and return stdout.""" + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout.strip() + + +def commit(repo: Path, message: str) -> str: + """Commit the fixture tree and return its SHA.""" + git(repo, "add", ".") + git(repo, "commit", "-m", message) + return git(repo, "rev-parse", "HEAD") + + +def fixture_repo(tmp_path: Path) -> tuple[Path, str, str]: + """Create a repository with one changed TypeScript runtime line.""" + repo = tmp_path / "repo" + source = repo / "src" / "calculate_total.ts" + source.parent.mkdir(parents=True) + git(repo, "init", "-b", "main") + git(repo, "config", "user.name", "Coverage Test") + git(repo, "config", "user.email", "coverage@example.invalid") + source.write_text( + "export function calculateTotal(left: number, right: number) {\n" + " return left + right;\n" + "}\n", + encoding="utf-8", + ) + base_sha = commit(repo, "base") + source.write_text( + "export function calculateTotal(left: number, right: number) {\n" + " return left - right;\n" + "}\n", + encoding="utf-8", + ) + head_sha = commit(repo, "change calculation") + return repo, base_sha, head_sha + + +def write_coverage( + repo: Path, + *, + statement_count: int, + include_source: bool = True, + branch_counts: list[int] | None = None, +) -> Path: + """Write Istanbul final and deliberately low global summary evidence.""" + coverage_dir = repo / "coverage" + coverage_dir.mkdir() + source = repo / "src" / "calculate_total.ts" + final_data = {} + if include_source: + final_data[str(source)] = { + "statementMap": { + "0": { + "start": {"line": 2, "column": 2}, + "end": {"line": 2, "column": 22}, + } + }, + "s": {"0": statement_count}, + "fnMap": { + "0": { + "name": "calculateTotal", + "decl": { + "start": {"line": 1, "column": 0}, + "end": {"line": 1, "column": 30}, + }, + "loc": { + "start": {"line": 1, "column": 0}, + "end": {"line": 3, "column": 1}, + }, + } + }, + "f": {"0": statement_count}, + "branchMap": ( + { + "0": { + "type": "if", + "locations": [ + { + "start": {"line": 2, "column": 2}, + "end": {"line": 2, "column": 22}, + }, + { + "start": {"line": 2, "column": 2}, + "end": {"line": 2, "column": 22}, + }, + ], + } + } + if branch_counts is not None + else {} + ), + "b": {"0": branch_counts} if branch_counts is not None else {}, + } + (coverage_dir / "coverage-final.json").write_text( + json.dumps(final_data), encoding="utf-8" + ) + (coverage_dir / "coverage-summary.json").write_text( + json.dumps( + { + "total": { + metric: {"pct": 42.0} + for metric in ("statements", "branches", "functions", "lines") + } + } + ), + encoding="utf-8", + ) + summary_list = repo / "coverage-files.txt" + summary_list.write_text( + "\ncoverage/coverage-summary.json\ncoverage/coverage-final.json\n", + encoding="utf-8", + ) + return summary_list + + +def run_gate(repo: Path, base_sha: str, head_sha: str, summary_list: Path) -> int: + """Run the coverage gate against one fixture repository.""" + return gate.main( + [ + "--repo-root", + str(repo), + "--base-sha", + base_sha, + "--head-sha", + head_sha, + "--summary-list", + str(summary_list), + ] + ) + + +def test_low_global_coverage_is_advisory_when_changed_units_are_covered( + tmp_path: Path, capsys +) -> None: + repo, base_sha, head_sha = fixture_repo(tmp_path) + summary_list = write_coverage(repo, statement_count=1) + + assert run_gate(repo, base_sha, head_sha, summary_list) == 0 + report = capsys.readouterr().out + assert "Global coverage advisory" in report + assert "statements: 42.0%" in report + assert "src/calculate_total.ts: statements 1/1" in report + assert "Result: PASS" in report + + +def test_uncovered_changed_statement_fails_with_file_and_metric( + tmp_path: Path, capsys +) -> None: + repo, base_sha, head_sha = fixture_repo(tmp_path) + summary_list = write_coverage(repo, statement_count=0) + + assert run_gate(repo, base_sha, head_sha, summary_list) == 1 + report = capsys.readouterr().out + assert "src/calculate_total.ts: statements 0/1" in report + assert "changed statements coverage is 0/1" in report + assert "Result: FAIL" in report + + +def test_partially_covered_changed_branch_fails(tmp_path: Path, capsys) -> None: + repo, base_sha, head_sha = fixture_repo(tmp_path) + summary_list = write_coverage( + repo, + statement_count=1, + branch_counts=[1, 0], + ) + + assert run_gate(repo, base_sha, head_sha, summary_list) == 1 + report = capsys.readouterr().out + assert "src/calculate_total.ts: statements 1/1, branches 1/2" in report + assert "changed branches coverage is 1/2" in report + assert "Result: FAIL" in report + + +def test_changed_runtime_source_missing_from_instrumentation_fails( + tmp_path: Path, capsys +) -> None: + repo, base_sha, head_sha = fixture_repo(tmp_path) + summary_list = write_coverage(repo, statement_count=0, include_source=False) + + assert run_gate(repo, base_sha, head_sha, summary_list) == 1 + report = capsys.readouterr().out + assert "src/calculate_total.ts is absent from coverage-final.json" in report + assert "Result: FAIL" in report + + +def test_test_only_change_has_explicit_not_applicable_result( + tmp_path: Path, capsys +) -> None: + repo = tmp_path / "repo" + test_file = repo / "src" / "calculate_total.test.ts" + test_file.parent.mkdir(parents=True) + git(repo, "init", "-b", "main") + git(repo, "config", "user.name", "Coverage Test") + git(repo, "config", "user.email", "coverage@example.invalid") + test_file.write_text("test('base', () => {});\n", encoding="utf-8") + base_sha = commit(repo, "base") + test_file.write_text("test('changed', () => {});\n", encoding="utf-8") + head_sha = commit(repo, "test only") + summary_list = write_coverage(repo, statement_count=0, include_source=False) + + assert run_gate(repo, base_sha, head_sha, summary_list) == 0 + report = capsys.readouterr().out + assert "No changed JavaScript/TypeScript runtime source files" in report + assert "Result: PASS" in report + + +@pytest.mark.parametrize( + "path", + [ + "README.md", + "tests/runtime.ts", + "src/runtime.d.ts", + "vite.config.ts", + ], +) +def test_non_runtime_javascript_paths_are_explicitly_excluded(path: str) -> None: + assert not gate.is_runtime_source(path) + + +def test_git_command_error_is_scrubbed_into_runtime_error(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="fatal:"): + gate.git(tmp_path, "status") + + +def test_changed_file_enumeration_error_is_visible(monkeypatch, tmp_path: Path) -> None: + completed = subprocess.CompletedProcess( + args=["git"], + returncode=1, + stdout=b"", + stderr=b"enumeration failed", + ) + monkeypatch.setattr(gate.subprocess, "run", lambda *args, **kwargs: completed) + + with pytest.raises(RuntimeError, match="enumeration failed"): + gate.changed_runtime_lines(tmp_path, "base", "head") + + +def test_invalid_istanbul_locations_do_not_intersect() -> None: + assert gate.location_range(None) is None + assert gate.location_range({"start": {"line": "two"}}) is None + assert gate.location_range({"start": {"line": 2}, "end": {}}) == (2, 2) + assert not gate.intersects(None, {2}) + assert gate.intersects( + {"start": {"line": 2}, "end": {"line": 4}}, + {4}, + ) + + +def test_unmapped_and_unchanged_istanbul_units_are_ignored() -> None: + invalid = {"start": {}, "end": {}} + unchanged = {"start": {"line": 10}, "end": {"line": 10}} + record = { + "statementMap": {"invalid": invalid, "unchanged": unchanged}, + "s": {"invalid": 1, "unchanged": 1}, + "fnMap": { + "invalid": {"loc": invalid}, + "unchanged": {"name": "unchanged", "loc": unchanged}, + }, + "f": {"invalid": 1, "unchanged": 1}, + "branchMap": { + "invalid": {"type": "if", "locations": [invalid]}, + "unchanged": {"type": "if", "loc": unchanged}, + }, + "b": {"invalid": [1], "unchanged": [1]}, + } + + assert gate.changed_metric_counts([record], {2}) == { + metric: (0, 0) for metric in gate.METRICS + } + + +def test_coverage_path_normalization_handles_relative_suffix_and_ambiguity( + tmp_path: Path, +) -> None: + changed = {"frontend/src/runtime.ts"} + assert ( + gate.normalize_coverage_path( + "./frontend/src/runtime.ts", tmp_path, changed + ) + == "frontend/src/runtime.ts" + ) + assert ( + gate.normalize_coverage_path( + "/different/root/frontend/src/runtime.ts", tmp_path, changed + ) + == "frontend/src/runtime.ts" + ) + assert ( + gate.normalize_coverage_path( + "/different/root/src/runtime.ts", + tmp_path, + {"frontend/src/runtime.ts", "backend/src/runtime.ts"}, + ) + is None + ) + assert ( + gate.normalize_coverage_path( + str(tmp_path.parent / "outside" / "not-a-match.ts"), + tmp_path, + {"frontend/src/runtime.ts"}, + ) + is None + ) + + +def test_runtime_line_classifier_distinguishes_types_comments_and_code( + tmp_path: Path, +) -> None: + source = tmp_path / "src" / "runtime.ts" + source.parent.mkdir() + source.write_text( + "\n" + "/* block\n" + " * comment\n" + " */\n" + "// line comment\n" + "interface RuntimeShape {\n" + "}\n" + "export type RuntimeId = string;\n" + "import type { Runtime } from './types';\n" + "const runtimeValue = 1;\n", + encoding="utf-8", + ) + + assert gate.likely_runtime_lines( + tmp_path, "src/runtime.ts", set(range(1, 11)) + ) == [10] + + +def test_missing_or_invalid_coverage_evidence_fails_closed( + tmp_path: Path, capsys +) -> None: + missing = tmp_path / "missing.txt" + assert gate.main( + [ + "--repo-root", + str(tmp_path), + "--base-sha", + "base", + "--head-sha", + "head", + "--summary-list", + str(missing), + ] + ) == 1 + assert "coverage evidence could not be evaluated" in capsys.readouterr().out + + +def test_changed_source_requires_coverage_final_json( + tmp_path: Path, capsys +) -> None: + repo, base_sha, head_sha = fixture_repo(tmp_path) + coverage_dir = repo / "coverage" + coverage_dir.mkdir() + summary = coverage_dir / "coverage-summary.json" + summary.write_text( + json.dumps({"total": {metric: {"pct": 100} for metric in gate.METRICS}}), + encoding="utf-8", + ) + summary_list = repo / "coverage-files.txt" + summary_list.write_text("coverage/coverage-summary.json\n", encoding="utf-8") + + assert run_gate(repo, base_sha, head_sha, summary_list) == 1 + assert "coverage-final.json is required" in capsys.readouterr().out + + +def test_runtime_looking_change_without_mapped_units_fails( + tmp_path: Path, capsys +) -> None: + repo, base_sha, head_sha = fixture_repo(tmp_path) + summary_list = write_coverage(repo, statement_count=1) + final_path = repo / "coverage" / "coverage-final.json" + final_path.write_text( + json.dumps( + { + str(repo / "src" / "calculate_total.ts"): { + "statementMap": {}, + "s": {}, + "fnMap": {}, + "f": {}, + "branchMap": {}, + "b": {}, + } + } + ), + encoding="utf-8", + ) + + assert run_gate(repo, base_sha, head_sha, summary_list) == 1 + assert "Istanbul mapped no execution units" in capsys.readouterr().out + + +def test_comment_only_change_without_mapped_units_passes( + tmp_path: Path, capsys +) -> None: + repo = tmp_path / "repo" + source = repo / "src" / "runtime.ts" + source.parent.mkdir(parents=True) + git(repo, "init", "-b", "main") + git(repo, "config", "user.name", "Coverage Test") + git(repo, "config", "user.email", "coverage@example.invalid") + source.write_text("// base comment\n", encoding="utf-8") + base_sha = commit(repo, "base") + source.write_text("// changed comment\n", encoding="utf-8") + head_sha = commit(repo, "comment only") + coverage_dir = repo / "coverage" + coverage_dir.mkdir() + (coverage_dir / "coverage-final.json").write_text( + json.dumps( + { + str(source): { + "statementMap": {}, + "s": {}, + "fnMap": {}, + "f": {}, + "branchMap": {}, + "b": {}, + } + } + ), + encoding="utf-8", + ) + summary_list = repo / "coverage-files.txt" + summary_list.write_text("coverage/coverage-final.json\n", encoding="utf-8") + + assert run_gate(repo, base_sha, head_sha, summary_list) == 0 + report = capsys.readouterr().out + assert "comments, delimiters, or type-only declarations" in report + assert "Result: PASS" in report diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index a38f74c0..cd9ccce4 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -87,6 +87,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs == [ ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], + ["github-models", "openai/gpt-4.1"], ["github-models", "openai/gpt-5"], ["github-models", "openai/gpt-5-chat"], ["github-models", "openai/o3"], @@ -97,6 +98,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert set(github_candidate_models).issubset(set(github_models)) assert github_candidate_models == [ "deepseek/deepseek-v3-0324", + "openai/gpt-4.1", "openai/gpt-5", "openai/gpt-5-chat", "openai/o3", @@ -137,6 +139,25 @@ def is_reasoning_capable(model_name: str) -> bool: assert "variants" not in model_config, model_name +def test_central_adversarial_harness_isolates_live_review_environment(): + """Focused regressions must not consume the parent review's live evidence.""" + runner = Path("scripts/ci/run_opencode_review_model_pool.sh").read_text( + encoding="utf-8" + ) + harness = runner.split("run_central_adversarial_harness()", 1)[1].split( + "fallback_without_model_catalog()", 1 + )[0] + + for name in ( + "OPENCODE_APPROVAL_REPAIR_EVIDENCE_FILE", + "OPENCODE_CHANGED_FILES_FILE", + "OPENCODE_DYNAMIC_REVIEW_CADENCE", + "OPENCODE_EVIDENCE_FILE", + "OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION", + ): + assert harness.count(f"-u {name}") == 2 + + def test_opencode_trusted_source_ref_is_not_controlled_by_workflow_inputs(): """Check out trusted source directly from the workflow identity SHA.""" workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") @@ -525,11 +546,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "CHECK_LOOKUP_GH_TOKEN" in workflow assert "CONFIGURED_REVIEW_WRITE_TOKEN_SOURCE" in workflow assert "retrying with workflow github token" in workflow - assert 'review_write_token="$OPENCODE_APP_TOKEN"' in workflow - assert 'review_write_token="$CHECK_LOOKUP_GH_TOKEN"' in workflow - assert 'review_write_token="$configured_review_write_token"' in workflow - assert 'review_write_fallback_token="$CHECK_LOOKUP_GH_TOKEN"' in workflow - assert "review write fallback token source=" in workflow + assert 'review_write_token="${OPENCODE_APP_TOKEN:-}"' in workflow + assert 'review_write_token="$CHECK_LOOKUP_GH_TOKEN"' not in workflow + assert 'review_write_token="$configured_review_write_token"' not in workflow + assert "review write fallback token source=disabled" in workflow assert "using github-token primary and opencode-app fallback" not in workflow assert 'review_write_token="${OPENCODE_APP_TOKEN:-$GH_TOKEN}"' not in workflow assert 'REVIEW_PUBLISH_RETRY_ATTEMPTS: "1"' in workflow @@ -537,7 +557,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "gh_error_is_retryable_publication_failure()" in workflow assert "review_publish_retry_sleep_seconds()" in workflow assert 'post_pull_review_with_retry "primary review"' in workflow - assert 'post_pull_review_with_retry "fallback review"' in workflow + assert 'post_pull_review_with_retry "fallback review"' not in workflow assert "GitHub review publication retry sleep capped from %s to %s seconds." in workflow assert "hit a retryable GitHub API throttle; retrying attempt" in workflow assert "GitHub returned HTTP 422 for this review write; likely causes are token/event policy" in workflow @@ -605,8 +625,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "ContextualWisdomLab/.github:code-reviewer-prompt.md | \\" in workflow assert "opencode.jsonc | \\" in workflow assert "ContextualWisdomLab/.github:.jules/bolt.md | \\" in workflow + assert "ContextualWisdomLab/.github:scripts/ci/javascript_coverage_gate.py | \\" in workflow assert "ContextualWisdomLab/.github:scripts/ci/opencode_review_approve_gate.sh | \\" in workflow assert "scripts/ci/run_opencode_review_model_pool.sh | \\" in workflow + assert "ContextualWisdomLab/.github:tests/test_javascript_coverage_gate.py | \\" in workflow assert "tests/test_opencode_agent_contract.py | \\" in workflow assert "ContextualWisdomLab/appguardrail:scripts/ci/collect_org_security_failures.py" in workflow assert "ContextualWisdomLab/appguardrail:.github/workflows/org-security-failure-collector.yml" in workflow @@ -626,12 +648,23 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert workflow.index("Detect central review-process scope") < workflow.index( "Initialize CodeGraph index for OpenCode" ) + assert "Install central adversarial harness runtime" in workflow + assert ( + workflow.index("Install central adversarial harness runtime") + < workflow.index("Run OpenCode PR Review model pool") + ) + assert ( + "steps.central_review_process_fallback_scope.outputs.eligible == 'true'" + in workflow + ) + assert "--only-binary=:all: -r requirements-opencode-review-ci-hashes.txt" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL" in workflow assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "120"' in workflow assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "180"' in workflow assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1"' in workflow assert "Central review-process evidence fallback eligible" in model_pool_runner + assert "hash-pinned uv runtime is not installed in the model-pool job" in model_pool_runner assert "provider delay is logged before the publish fallback evaluates current-head peer evidence" in model_pool_runner assert "model pool was intentionally skipped" not in workflow assert "current-head deterministic evidence is clean" in workflow @@ -672,15 +705,15 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert re.search(r"Prepare bounded OpenCode review evidence[\s\S]{0,120}timeout-minutes: 12", workflow) assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 45", workflow) assert 'timeout-minutes: 12' in workflow - assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 20", workflow) - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "900"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "1080"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "900"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "300"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "600"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "1080"' in workflow - assert 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-1080}s"' in workflow + assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 12", workflow) + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "180"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "540"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "540"' in workflow + assert 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-540}s"' in workflow assert "OpenCode model pool exceeded the outer" in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert re.search(r"Run OpenCode PR Review model pool[\s\S]{0,280}continue-on-error: true", workflow) @@ -693,6 +726,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert ( 'OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 ' "openai/gpt-5.6-luna " + "github-models/openai/gpt-4.1 " "github-models/openai/gpt-5 " "github-models/openai/gpt-5-chat " "github-models/openai/o3 " @@ -700,21 +734,22 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'github-models/deepseek/deepseek-r1"' ) in workflow assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "300"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' in workflow assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "600"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "1080"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "540"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "540"' in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_DYNAMIC_REVIEW_CADENCE: "true"' in workflow assert 'OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt' in workflow - assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "300"' in workflow - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "420"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "900"' in workflow - assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "420"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "1080"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "420"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "900"' in workflow + assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "90"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "180"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "120"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow + assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "180"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "540"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "120"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "360"' in workflow + assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow publish_step = workflow.split(" - name: Publish OpenCode review outcome", 1)[1].split( @@ -759,6 +794,13 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "do not request changes solely because your own tool or file read did not" in workflow assert "while :" in model_pool_runner assert "should_skip_model_candidate" in model_pool_runner + assert "cap_model_run_timeout" in model_pool_runner + assert "constrained request-body limit" in model_pool_runner + assert "run_central_adversarial_harness" in model_pool_runner + assert "finish_pool_without_model" in model_pool_runner + assert "current-head CodeGraph index is missing or empty" in model_pool_runner + assert "general repository reviews still fail closed" in model_pool_runner + assert "pull-request-target-gitlink-is-explicitly-skipped" in model_pool_runner assert "is_low_sensitivity_candidate" in model_pool_runner assert "mini/nano review models are disabled" in model_pool_runner assert "OPENAI_API_KEY is not configured" in model_pool_runner @@ -779,6 +821,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert ( 'OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 ' "openai/gpt-5.6-luna " + "github-models/openai/gpt-4.1 " "github-models/openai/gpt-5 " "github-models/openai/gpt-5-chat " "github-models/openai/o3 " @@ -984,16 +1027,33 @@ def test_opencode_review_publication_prefers_app_token_for_review_writes(): ) assert ( - "GH_TOKEN: ${{ steps.opencode_app_token.outputs.token || " - "secrets.PR_REVIEW_MERGE_TOKEN || " - "secrets.OPENCODE_APPROVE_TOKEN || github.token }}" + "GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }}" ) in workflow assert ( "CONFIGURED_REVIEW_WRITE_TOKEN_SOURCE: ${{ steps.opencode_app_token.outputs.available == 'true' && " "'opencode-app' || secrets.PR_REVIEW_MERGE_TOKEN" ) in workflow - assert 'review_write_token="$OPENCODE_APP_TOKEN"' in workflow - assert 'review_write_token="$CHECK_LOOKUP_GH_TOKEN"' in workflow + assert 'review_write_token="${OPENCODE_APP_TOKEN:-}"' in workflow + assert 'post_pull_review_with_retry "fallback review"' not in workflow + assert "OPENCODE_REVIEW_IDENTITY_UNAVAILABLE" in workflow + assert "OPENCODE_REVIEW_STALE_HEAD" in workflow + assert "OPENCODE_OVERVIEW_STALE_HEAD" in workflow + assert "review_live_head_sha()" in workflow + assert "validate_published_review_head()" in workflow + assert "dismiss_stale_published_review()" in workflow + assert 'review_head_guard_token="${GH_TOKEN:-$review_write_token}"' in workflow + assert 'post_pull_review_with_retry "primary review" "$review_write_token" "$review_payload_file" "$gh_error_file" "$review_response_file"' in workflow + assert 'post_pull_review_with_retry "inline review" "$review_write_token" "$review_payload_file" "$gh_error_file" "$review_response_file"' in workflow + assert 'reviews/${review_id}/dismissals' in workflow + assert "CENTRAL_FAST_APPROVAL_STALE_HEAD" in workflow + assert ( + 'select(.user.login == "opencode-agent[bot]" and ' + '(.body | contains("")))' + ) in workflow + assert ( + 'select((.user.login == "github-actions[bot]" or ' + '.user.login == "opencode-agent[bot]")' + ) not in workflow assert 'OPENCODE_APP_TOKEN_EXCHANGE_TIMEOUT_SECONDS: "20"' in workflow assert '--max-time "${OPENCODE_APP_TOKEN_EXCHANGE_TIMEOUT_SECONDS}"' in workflow assert "app token request did not complete within ${OPENCODE_APP_TOKEN_EXCHANGE_TIMEOUT_SECONDS}s" in workflow @@ -1020,6 +1080,13 @@ def test_opencode_approve_review_publication_failure_keeps_gate_result(): ) assert "Publish central OpenCode fast approval" in workflow assert "steps.central_fast_approval.outputs.published != 'true'" in workflow + fast_approval = workflow.split(" - name: Publish central OpenCode fast approval", 1)[ + 1 + ].split(" - name: Publish OpenCode review outcome", 1)[0] + assert "continue-on-error: true" in fast_approval + assert "def latest_peer_checks:" in fast_approval + assert 'group_by([.app.slug // "", .name // ""])' in fast_approval + assert fast_approval.count("latest_peer_checks") == 3 assert "CENTRAL_FAST_APPROVAL_WAITING_FOR_CHECKS" in workflow assert "CENTRAL_FAST_APPROVAL_CODE_SCANNING_ALERTS" in workflow assert "Central fast approval published APPROVE review" in workflow diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 6f4eebe5..3a2b3c26 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import re import shutil @@ -15,6 +16,15 @@ ROOT = Path(__file__).resolve().parents[1] RUNNER = ROOT / "scripts" / "ci" / "run_opencode_review_model_pool.sh" +CENTRAL_FALLBACK_ENV = { + "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE", + "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL", + "OPENCODE_APPROVAL_REPAIR_EVIDENCE_FILE", + "OPENCODE_DYNAMIC_REVIEW_CADENCE", + "OPENCODE_EVIDENCE_FILE", + "OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION", +} + def bash_command() -> str: """Return a Bash executable that can run repository shell scripts locally.""" @@ -108,6 +118,8 @@ def run_failed_model( fake_opencode.chmod(0o755) github_output = tmp_path / "github-output.txt" env = os.environ.copy() + for name in CENTRAL_FALLBACK_ENV: + env.pop(name, None) env.update( { "FAKE_OPENCODE_JSON": json_line, @@ -147,6 +159,146 @@ def run_failed_model( ) +def run_central_fallback( + tmp_path: Path, + *, + changed_files: list[str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], Path, Path, Path]: + """Run the bounded central fallback with deterministic local command fixtures.""" + command = bash_command() + skip_if_windows_bash_is_unresponsive(command) + source_dir = tmp_path / "source" + review_dir = tmp_path / "review" + runner_temp = tmp_path / "runner-temp" + fake_bin = tmp_path / "bin" + for path in ( + source_dir / ".codegraph", + source_dir / "scripts" / "ci", + source_dir / "tests", + review_dir, + runner_temp, + fake_bin, + ): + path.mkdir(parents=True, exist_ok=True) + + (source_dir / ".codegraph" / "codegraph.db").write_bytes(b"indexed") + (source_dir / "scripts" / "ci" / "run_opencode_review_model_pool.sh").write_text( + "#!/usr/bin/env bash\ncap_model_run_timeout() { :; }\n", + encoding="utf-8", + ) + (source_dir / "scripts" / "ci" / "javascript_coverage_gate.py").write_text( + "def normalize_coverage_path():\n return None\n", + encoding="utf-8", + ) + (source_dir / "scripts" / "ci" / "strix_quick_gate.sh").write_text( + "#!/usr/bin/env bash\nmode=160000\n", + encoding="utf-8", + ) + strix_test = source_dir / "scripts" / "ci" / "test_strix_quick_gate.sh" + strix_test.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "test \"${STRIX_TEST_CASE_FILTER:-}\" = " + "pull-request-target-gitlink-is-explicitly-skipped\n" + "printf 'pull-request-target-gitlink-is-explicitly-skipped: PASS\\n'\n", + encoding="utf-8", + ) + strix_test.chmod(0o755) + + uv_log = tmp_path / "uv.log" + fake_uv = fake_bin / "uv" + fake_uv.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "printf '%s\\n' \"$*\" > \"${FAKE_UV_LOG:?}\"\n" + "printf 'focused pytest: PASS\\n'\n", + encoding="utf-8", + ) + fake_uv.chmod(0o755) + + required_paths = [ + "scripts/ci/run_opencode_review_model_pool.sh", + "scripts/ci/javascript_coverage_gate.py", + "scripts/ci/strix_quick_gate.sh", + ] + changed_files_file = tmp_path / "changed-files.txt" + changed_files_file.write_text( + "\n".join(required_paths if changed_files is None else changed_files) + "\n", + encoding="utf-8", + ) + output_file = tmp_path / "selected-output.json" + github_output = tmp_path / "github-output.txt" + env = os.environ.copy() + for name in CENTRAL_FALLBACK_ENV: + env.pop(name, None) + env.update( + { + "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE": "true", + "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL": "central-review-process", + "FAKE_UV_LOG": bash_path(uv_log), + "GITHUB_OUTPUT": bash_path(github_output), + "GITHUB_WORKSPACE": bash_path(ROOT), + "HEAD_SHA": "2" * 40, + "OPENCODE_CHANGED_FILES_FILE": bash_path(changed_files_file), + "OPENCODE_MODEL_CANDIDATES": "", + "OPENCODE_OUTPUT_FILE": bash_path(output_file), + "OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION": "true", + "OPENCODE_REVIEW_WORKDIR": bash_path(review_dir), + "OPENCODE_SOURCE_WORKDIR": bash_path(source_dir), + "PATH": f"{bash_path(fake_bin)}:{env['PATH']}", + "RUNNER_TEMP": bash_path(runner_temp), + "RUN_ATTEMPT": "1", + "RUN_ID": "central-fallback-test", + } + ) + result = subprocess.run( + [command, bash_path(RUNNER)], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + return result, output_file, github_output, uv_log + + +def test_central_fallback_emits_structured_adversarial_approval(tmp_path: Path) -> None: + """Central self-repair can approve only after all bounded probes execute.""" + result, output_file, github_output, uv_log = run_central_fallback(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert "valid current-head APPROVE control block" in result.stdout + assert "review_model=central-current-head-adversarial-harness" in github_output.read_text( + encoding="utf-8" + ) + assert "review_status=success" in github_output.read_text(encoding="utf-8") + assert "test_github_gpt5_runtime_cap_preserves_queue_budget" in uv_log.read_text( + encoding="utf-8" + ) + control = json.loads(output_file.read_text(encoding="utf-8")) + assert control["result"] == "APPROVE" + assert control["adversarial_validation"]["status"] == "passed" + assert len(control["adversarial_validation"]["probes"]) == 3 + assert {probe["outcome"] for probe in control["adversarial_validation"]["probes"]} == { + "falsified" + } + + +def test_central_fallback_fails_closed_when_required_scope_is_missing(tmp_path: Path) -> None: + """A central-looking change cannot use the harness without every reviewed core path.""" + result, output_file, github_output, uv_log = run_central_fallback( + tmp_path, + changed_files=["scripts/ci/run_opencode_review_model_pool.sh"], + ) + + assert result.returncode == 1 + assert "required current-head path scripts/ci/javascript_coverage_gate.py is not changed" in result.stdout + assert "review_status=exhausted" in github_output.read_text(encoding="utf-8") + assert output_file.read_text(encoding="utf-8") == "" + assert not uv_log.exists() + + def test_failed_provider_logs_bounded_reason_and_redacts_credentials(tmp_path: Path) -> None: """Provider JSON/stderr reasons remain useful without leaking credentials.""" fake_bearer_token = "secret" + "-value" @@ -269,6 +421,32 @@ def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> Non assert "retry budget remaining." in result.stdout +def test_github_gpt5_runtime_cap_preserves_queue_budget(tmp_path: Path) -> None: + """Known constrained GitHub GPT-5 endpoints cannot consume a full cadence slot.""" + result = run_failed_model( + tmp_path, + extra_env={ + "OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS": "3", + "OPENCODE_RUN_TIMEOUT_SECONDS": "9", + }, + ) + + assert result.returncode == 1 + assert ( + "OpenCode github-models/openai/gpt-5 runtime cap selected 3s instead of 9s " + "because this installation has returned a constrained request-body limit for that endpoint." + ) in result.stdout + attempt_budget = re.search( + r"OpenCode github-models/openai/gpt-5 attempt 1/1 using (\d+)s run timeout " + r"with (\d+)s retry budget remaining\.", + result.stdout, + ) + assert attempt_budget is not None + run_timeout, remaining_budget = map(int, attempt_budget.groups()) + assert run_timeout == 3 + assert run_timeout <= remaining_budget <= 30 + + def test_github_models_openai_prompt_references_evidence_without_inlining(tmp_path: Path) -> None: """Small-request GitHub Models OpenAI candidates keep evidence as files.""" prompt_capture = tmp_path / "captured-prompt.md"