hotfix(ci): render fenced evidence snippets in ai review comments

2026-09-02 08:19:59 +00:00 · 2026-04-02 00:44:44 -04:00
parent 164a8af82a
commit 07b4275543
5 changed files with 271 additions and 0 deletions
@@ -168,6 +168,47 @@ describe('normalize-ai-review-output', () => {
    expect(markdown).toContain('`old_marker_path`');
  });

+  test('renders finding snippets as renderer-owned fenced code blocks', () => {
+    const validation = reviewOutput.normalizeStructuredOutput(
+      JSON.stringify({
+        summary: 'One workflow branch still uses the stale marker path.',
+        findings: [
+          {
+            severity: 'medium',
+            title: 'Fallback branch still writes the stale marker file',
+            file: '.github/workflows/ai-review.yml',
+            line: 181,
+            what: 'One branch still writes the old marker file path.',
+            why: 'That can leave duplicate bot comments on reruns for the same PR SHA.',
+            fix: 'Keep the rerun marker keyed to PR plus head SHA in every publish branch.',
+            snippets: [
+              {
+                label: 'Current publish branch',
+                language: 'bash',
+                code: 'marker_file=\"$RUNNER_TEMP/.ai-review-marker\"\nprintf \"%s\\n\" \"$REVIEW_MARKER\" > \"$marker_file\"',
+              },
+            ],
+          },
+        ],
+        securityChecklist: [{ check: 'Workflow safety', status: 'pass', notes: 'Covered.' }],
+        ccsCompliance: [{ rule: 'Renderer-owned markdown', status: 'pass', notes: 'Covered.' }],
+        informational: [],
+        strengths: [],
+        overallAssessment: 'approved_with_notes',
+        overallRationale: 'This is a deterministic formatting-only follow-up.',
+      })
+    );
+
+    expect(validation.ok).toBe(true);
+    const markdown = reviewOutput.renderStructuredReview(validation.value, { model: 'glm-5.1' });
+
+    expect(markdown).toContain('Evidence: Current publish branch');
+    expect(markdown).toContain('```bash');
+    expect(markdown).toContain('marker_file="$RUNNER_TEMP/.ai-review-marker"');
+    expect(markdown).toContain('printf "%s\\n" "$REVIEW_MARKER" > "$marker_file"');
+    expect(markdown).toContain('```');
+  });
+
  test('normalizes optional rendering metadata when present in structured output', () => {
    const validation = reviewOutput.normalizeStructuredOutput(
      JSON.stringify({
@@ -520,6 +561,41 @@ describe('normalize-ai-review-output', () => {
    expect(validation.reason).toContain('securityChecklist must contain at least 1 item');
  });

+  test('rejects finding snippets that exceed the renderer snippet budget', () => {
+    const validation = reviewOutput.normalizeStructuredOutput(
+      JSON.stringify({
+        summary: 'The renderer should reject oversized snippet payloads.',
+        findings: [
+          {
+            severity: 'low',
+            title: 'Oversized snippet',
+            file: 'scripts/github/normalize-ai-review-output.mjs',
+            line: 1,
+            what: 'The example snippet is intentionally too long.',
+            why: 'Oversized snippets would bloat the published review comment.',
+            fix: 'Keep snippets short and renderer-owned.',
+            snippets: [
+              {
+                label: 'Too long',
+                language: 'txt',
+                code: Array.from({ length: 21 }, (_, index) => `line ${index + 1}`).join('\n'),
+              },
+            ],
+          },
+        ],
+        securityChecklist: [{ check: 'Injection safety', status: 'pass', notes: 'Covered.' }],
+        ccsCompliance: [{ rule: 'Renderer-owned markdown', status: 'pass', notes: 'Covered.' }],
+        informational: [],
+        strengths: [],
+        overallAssessment: 'approved_with_notes',
+        overallRationale: 'Oversized snippets should fail validation.',
+      })
+    );
+
+    expect(validation.ok).toBe(false);
+    expect(validation.reason).toContain('findings[0].snippets[0].code exceeds 20 lines');
+  });
+
  test('allows plain prose that references section labels without starting with them', () => {
    const validation = reviewOutput.normalizeStructuredOutput(
      JSON.stringify({
@@ -156,6 +156,82 @@ describe('run-ai-review-direct', () => {
    });
  });

+  test('renders finding snippets from the validated direct review response', async () => {
+    await withTempDir('ai-review-direct-', async (tempDir) => {
+      const outputFile = path.join(tempDir, 'review.md');
+      const logFile = path.join(tempDir, 'attempts.json');
+      const packetFile = path.join(tempDir, 'packet.md');
+      const manifestFile = path.join(tempDir, 'selected-files.txt');
+      const includedManifestFile = path.join(tempDir, 'included-files.txt');
+      fs.writeFileSync(packetFile, '# AI Review Packet\n\npacket body\n');
+      fs.writeFileSync(manifestFile, '.github/workflows/ai-review.yml\n');
+      fs.writeFileSync(includedManifestFile, '.github/workflows/ai-review.yml\n');
+
+      const result = await directReview.writeDirectReviewFromEnv(
+        {
+          ANTHROPIC_BASE_URL: 'https://api.z.ai/api/anthropic',
+          ANTHROPIC_AUTH_TOKEN: 'test-token',
+          REVIEW_MODEL: 'glm-5.1',
+          GITHUB_REPOSITORY: 'kaitranntt/ccs',
+          AI_REVIEW_PROMPT: 'You are a reviewer.',
+          AI_REVIEW_PACKET_FILE: packetFile,
+          AI_REVIEW_SCOPE_MANIFEST_FILE: manifestFile,
+          AI_REVIEW_PACKET_INCLUDED_MANIFEST_FILE: includedManifestFile,
+          AI_REVIEW_OUTPUT_FILE: outputFile,
+          AI_REVIEW_LOG_FILE: logFile,
+          AI_REVIEW_RUN_URL: 'https://github.com/kaitranntt/ccs/actions/runs/1',
+          AI_REVIEW_MODE: 'fast',
+          AI_REVIEW_SELECTED_FILES: '1',
+          AI_REVIEW_REVIEWABLE_FILES: '1',
+          AI_REVIEW_SELECTED_CHANGES: '24',
+          AI_REVIEW_REVIEWABLE_CHANGES: '24',
+          AI_REVIEW_SCOPE_LABEL: 'reviewable files',
+          AI_REVIEW_PACKET_INCLUDED_FILES: '1',
+          AI_REVIEW_PACKET_TOTAL_FILES: '1',
+          AI_REVIEW_PACKET_OMITTED_FILES: '0',
+          AI_REVIEW_TIMEOUT_MINUTES: '8',
+          AI_REVIEW_PR_NUMBER: '888',
+        },
+        async () =>
+          createResponse(
+            JSON.stringify({
+              summary: 'One non-blocking follow-up remains.',
+              findings: [
+                {
+                  severity: 'low',
+                  title: 'Marker write path still has one stale branch',
+                  file: '.github/workflows/ai-review.yml',
+                  line: 181,
+                  what: 'One branch still writes the stale marker file path.',
+                  why: 'That can make rerun behavior harder to reason about.',
+                  fix: 'Keep every publish branch aligned on the PR plus SHA marker.',
+                  snippets: [
+                    {
+                      label: 'Current branch body',
+                      language: 'bash',
+                      code: 'marker_file=\"$RUNNER_TEMP/.ai-review-marker\"\nprintf \"%s\\n\" \"$REVIEW_MARKER\" > \"$marker_file\"',
+                    },
+                  ],
+                },
+              ],
+              securityChecklist: [{ check: 'Injection safety', status: 'pass', notes: 'Covered.' }],
+              ccsCompliance: [{ rule: 'Renderer-owned markdown', status: 'pass', notes: 'Covered.' }],
+              informational: [],
+              strengths: ['The response validated on the first attempt.'],
+              overallAssessment: 'approved_with_notes',
+              overallRationale: 'The remaining change is formatter polish only.',
+            })
+          )
+      );
+
+      expect(result.usedFallback).toBe(false);
+      const markdown = fs.readFileSync(outputFile, 'utf8');
+      expect(markdown).toContain('Evidence: Current branch body');
+      expect(markdown).toContain('```bash');
+      expect(markdown).toContain('marker_file="$RUNNER_TEMP/.ai-review-marker"');
+    });
+  });
+
  test('retries with a repair attempt when the first response is invalid', async () => {
    await withTempDir('ai-review-direct-', async (tempDir) => {
      const outputFile = path.join(tempDir, 'review.md');