From 3ab1dfad08ce62254b49017825dd6004e5f6af33 Mon Sep 17 00:00:00 2001
From: Cole McIntosh <82463175+colesmcintosh@users.noreply.github.com>
Date: Mon, 23 Jun 2025 10:23:27 -0600
Subject: [PATCH] Add GitHub Actions workflow for LLM translation testing
artifacts (#11780)
* Add GitHub Actions workflow for LLM translation testing artifacts
* Update LLM translation testing workflow to fetch results from CircleCI and improve timeout settings. The job name has been changed for clarity, and the installation of dependencies has been replaced with CircleCI CLI setup. Placeholder test results are created if no CircleCI artifacts are found.
---
.github/workflows/llm-translation-testing.yml | 157 ++++++++++++++++++
1 file changed, 157 insertions(+)
create mode 100644 .github/workflows/llm-translation-testing.yml
diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml
new file mode 100644
index 0000000000..83566d6eb6
--- /dev/null
+++ b/.github/workflows/llm-translation-testing.yml
@@ -0,0 +1,157 @@
+name: LLM Translation Test Results for Release Candidates
+
+on:
+ workflow_dispatch:
+ inputs:
+ release_candidate_tag:
+ description: 'Release candidate tag/version'
+ required: true
+ type: string
+ push:
+ tags:
+ - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1
+
+jobs:
+ fetch-llm-translation-results:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
+
+ - name: Install CircleCI CLI
+ run: |
+ curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash
+
+ - name: Create test results directory
+ run: mkdir -p test-results
+
+ - name: Fetch LLM Translation Test Results from CircleCI
+ env:
+ CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }}
+ run: |
+ # Get the latest CircleCI pipeline for this commit
+ COMMIT_SHA="${{ github.sha }}"
+ echo "Fetching CircleCI results for commit: $COMMIT_SHA"
+
+ # Get pipeline info
+ PIPELINE_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
+ "https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline?branch=main" | \
+ jq -r ".items[] | select(.vcs.revision == \"$COMMIT_SHA\") | .id" | head -1)
+
+ if [ -z "$PIPELINE_INFO" ]; then
+ echo "No CircleCI pipeline found for commit $COMMIT_SHA"
+ echo "Creating placeholder test results..."
+ echo '' > test-results/junit.xml
+ echo '' >> test-results/junit.xml
+ echo 'No CircleCI results found for this commit' >> test-results/junit.xml
+ echo '' >> test-results/junit.xml
+ exit 0
+ fi
+
+ echo "Found pipeline: $PIPELINE_INFO"
+
+ # Get workflow info
+ WORKFLOW_ID=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
+ "https://circleci.com/api/v2/pipeline/$PIPELINE_INFO/workflow" | \
+ jq -r '.items[] | select(.name | contains("test")) | .id' | head -1)
+
+ if [ -z "$WORKFLOW_ID" ]; then
+ echo "No test workflow found in pipeline"
+ exit 1
+ fi
+
+ echo "Found workflow: $WORKFLOW_ID"
+
+ # Get job info for llm_translation tests
+ JOB_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
+ "https://circleci.com/api/v2/workflow/$WORKFLOW_ID/job" | \
+ jq -r '.items[] | select(.name | contains("llm_translation")) | select(.status == "success" or .status == "failed") | .job_number' | head -1)
+
+ if [ -z "$JOB_INFO" ]; then
+ echo "No completed llm_translation job found"
+ exit 1
+ fi
+
+ echo "Found job: $JOB_INFO"
+
+ # Download artifacts
+ curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
+ "https://circleci.com/api/v2/project/github/BerriAI/litellm/$JOB_INFO/artifacts" | \
+ jq -r '.items[] | select(.path | contains("junit") or contains("coverage") or contains("report")) | .url' | \
+ while read -r artifact_url; do
+ filename=$(basename "$artifact_url" | sed 's/[?&].*//')
+ echo "Downloading artifact: $filename"
+ curl -s -H "Circle-Token: $CIRCLE_TOKEN" -o "test-results/$filename" "$artifact_url"
+ done
+
+ # If no artifacts found, create placeholder
+ if [ ! -f "test-results/junit.xml" ]; then
+ echo "No test artifacts found, creating placeholder..."
+ echo '' > test-results/junit.xml
+ echo '' >> test-results/junit.xml
+ echo 'Test artifacts not available from CircleCI' >> test-results/junit.xml
+ echo '' >> test-results/junit.xml
+ fi
+ continue-on-error: true
+
+ - name: Generate test summary
+ run: |
+ echo "# LLM Translation Testing Results" > test-results/summary.md
+ echo "" >> test-results/summary.md
+ echo "**Release Candidate:** ${{ github.event.inputs.release_candidate_tag || github.ref_name }}" >> test-results/summary.md
+ echo "**Fetched Date:** $(date)" >> test-results/summary.md
+ echo "**Commit:** ${{ github.sha }}" >> test-results/summary.md
+ echo "**Source:** CircleCI Pipeline" >> test-results/summary.md
+ echo "" >> test-results/summary.md
+
+ # Parse junit.xml for test statistics if it exists
+ if [ -f "test-results/junit.xml" ]; then
+ python -c "
+ import xml.etree.ElementTree as ET
+ try:
+ tree = ET.parse('test-results/junit.xml')
+ root = tree.getroot()
+ tests = root.get('tests', '0')
+ failures = root.get('failures', '0')
+ errors = root.get('errors', '0')
+ skipped = root.get('skipped', '0')
+ time = root.get('time', '0')
+
+ print(f'**Total Tests:** {tests}')
+ print(f'**Passed:** {int(tests) - int(failures) - int(errors) - int(skipped)}')
+ print(f'**Failed:** {failures}')
+ print(f'**Errors:** {errors}')
+ print(f'**Skipped:** {skipped}')
+ print(f'**Duration:** {time} seconds')
+ except Exception as e:
+ print(f'Could not parse test results: {e}')
+ " >> test-results/summary.md
+ fi
+
+ echo "" >> test-results/summary.md
+ echo "## Test Files Covered" >> test-results/summary.md
+ ls tests/llm_translation/*.py | sed 's/^/- /' >> test-results/summary.md
+
+ - name: Upload test results
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: llm-translation-test-results-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+ path: |
+ test-results/
+ coverage.xml
+ htmlcov/
+ .coverage
+ retention-days: 30
+
+ - name: Upload JUnit test results
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: junit-xml-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+ path: test-results/junit.xml
+ retention-days: 30
\ No newline at end of file