fix(ci): harden ai review output

- replace raw assistant-text fallback with structured output normalization

- simplify the review prompt toward a tighter findings-only contract

- add tests for malformed output, safe fallback, and markdown escaping
This commit is contained in:
Tam Nhu Tran
2026-03-30 14:22:59 -04:00
parent 69cbb77924
commit 4950be7fc0
4 changed files with 375 additions and 165 deletions
+21 -41
View File
@@ -151,6 +151,8 @@ jobs:
MAX_THINKING_TOKENS: '16000'
REVIEW_OUTPUT_FILE: pr_review.md
REVIEW_COMMENT_FILE: .ccs-ai-review-comment.md
REVIEW_OUTPUT_SCHEMA: >-
{"type":"object","additionalProperties":false,"properties":{"summary":{"type":"string","minLength":1},"findings":{"type":"array","maxItems":6,"items":{"type":"object","additionalProperties":false,"properties":{"severity":{"type":"string","enum":["high","medium","low"]},"title":{"type":"string","minLength":1},"file":{"type":"string","minLength":1},"line":{"type":["integer","null"],"minimum":1},"what":{"type":"string","minLength":1},"why":{"type":"string","minLength":1},"fix":{"type":"string","minLength":1}},"required":["severity","title","file","what","why","fix"]}},"overallAssessment":{"type":"string","enum":["approved","approved_with_notes","changes_requested"]},"overallRationale":{"type":"string","minLength":1},"notes":{"type":"array","maxItems":4,"items":{"type":"string","minLength":1}}},"required":["summary","findings","overallAssessment","overallRationale"]}
steps:
- name: Prepare isolated Claude runtime
@@ -224,7 +226,8 @@ jobs:
anthropic_api_key: ${{ secrets.GLM_API_KEY }}
github_token: ${{ steps.app-token.outputs.token }}
allowed_non_write_users: ${{ needs.prepare.outputs.contributor_source == 'external' && '*' || '' }}
show_full_output: true # Visible logs for debugging slow/failing reviews
display_report: false # Keep all public review output on the normalized comment path
show_full_output: false # Keep scratch output out of public logs
track_progress: false # Disabled - no progress comments, just final review
prompt: |
think
@@ -242,55 +245,32 @@ jobs:
${{ steps.review-prompt.outputs.content }}
## IMPORTANT: Writing the Review
After completing your analysis, use the `Write` tool to write the final review markdown to `${{ env.REVIEW_OUTPUT_FILE }}`.
Do NOT use `Edit` tool — use `Write` tool directly to create the file in one shot.
Do NOT post any GitHub comments yourself. The workflow will publish the saved file.
Do NOT modify any source code files — this is a READ-ONLY review.
End your review with:
> 🤖 Reviewed by `${{ env.REVIEW_MODEL }}`
IMPORTANT RULES:
- Use `Write` tool to overwrite `${{ env.REVIEW_OUTPUT_FILE }}` with the complete review
- Do NOT use shell operators like || or && in bash commands
- Do NOT use heredoc (<<) syntax in bash commands
- Use simple, single-purpose bash commands only
## Runtime Rules
- This is a READ-ONLY review. Do not edit files.
- Use the checked-out PR branch plus surrounding repository context before reporting a finding.
- Return only structured output that matches the provided JSON schema.
- Do NOT write files.
- Do NOT post GitHub comments yourself.
- If no confirmed issues remain, return an empty findings array instead of inventing low-value feedback.
claude_args: |
--bare
--model ${{ env.REVIEW_MODEL }}
--permission-mode bypassPermissions
--max-turns 40
--allowedTools "Glob,Grep,Read,Write,Bash(gh pr diff *),Bash(gh pr view *),Bash(git diff *),Bash(git log *),Bash(git show *),Bash(cat *),Bash(ls *),Bash(wc *),Bash(head *),Bash(tail *),Bash(find *)"
--allowedTools "Read,Bash(gh pr diff:*),Bash(gh pr view:*),Bash(git diff:*),Bash(git log:*),Bash(git show:*),Bash(cat:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(find:*),Bash(grep:*)"
--json-schema '${{ env.REVIEW_OUTPUT_SCHEMA }}'
# Fallback: if Claude didn't write the review file, extract from execution output
- name: Extract review from execution output (fallback)
- name: Render review comment
if: always() && steps.claude-review.outcome != 'cancelled'
run: |
EXEC_LOG="$RUNNER_TEMP/claude-execution-output.json"
if [ -s "$REVIEW_OUTPUT_FILE" ]; then
echo "[i] Review file exists, skipping fallback extraction"
exit 0
fi
if [ ! -f "$EXEC_LOG" ]; then
echo "::warning::No execution output found at $EXEC_LOG"
exit 0
fi
# Extract last assistant text message as fallback review
EXTRACTED=$(jq -r '
[.[] | select(.type == "assistant") | .message.content[]?
| select(.type == "text") | .text] | last // empty
' "$EXEC_LOG" 2>/dev/null || true)
if [ -z "$EXTRACTED" ]; then
echo "::warning::Could not extract review content from execution output"
printf '## AI Review (incomplete)\n\nClaude completed but did not produce a structured review.\nCheck the [execution log artifact](%s) for details.\n\n> Reviewed by `%s` (fallback extraction)\n' \
"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
"$REVIEW_MODEL" > "$REVIEW_OUTPUT_FILE"
else
printf '%s\n' "$EXTRACTED" > "$REVIEW_OUTPUT_FILE"
fi
echo "[i] Fallback review extracted from execution output"
node scripts/github/normalize-ai-review-output.mjs
env:
AI_REVIEW_EXECUTION_FILE: ${{ runner.temp }}/claude-execution-output.json
AI_REVIEW_MODEL: ${{ env.REVIEW_MODEL }}
AI_REVIEW_OUTPUT_FILE: ${{ env.REVIEW_OUTPUT_FILE }}
AI_REVIEW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
AI_REVIEW_STRUCTURED_OUTPUT: ${{ steps.claude-review.outputs.structured_output }}
- name: Publish review comment
if: always() && steps.claude-review.outcome != 'cancelled'