mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-14 16:25:29 +00:00
Add GitHub Actions workflow for LLM translation testing artifacts (#11780)
* Add GitHub Actions workflow for LLM translation testing artifacts * Update LLM translation testing workflow to fetch results from CircleCI and improve timeout settings. The job name has been changed for clarity, and the installation of dependencies has been replaced with CircleCI CLI setup. Placeholder test results are created if no CircleCI artifacts are found.
This commit is contained in:
Cole McIntosh
@@ -0,0 +1,157 @@
|
||||
name: LLM Translation Test Results for Release Candidates
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_candidate_tag:
|
||||
description: 'Release candidate tag/version'
|
||||
required: true
|
||||
type: string
|
||||
push:
|
||||
tags:
|
||||
- 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1
|
||||
|
||||
jobs:
|
||||
fetch-llm-translation-results:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
|
||||
|
||||
- name: Install CircleCI CLI
|
||||
run: |
|
||||
curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash
|
||||
|
||||
- name: Create test results directory
|
||||
run: mkdir -p test-results
|
||||
|
||||
- name: Fetch LLM Translation Test Results from CircleCI
|
||||
env:
|
||||
CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }}
|
||||
run: |
|
||||
# Get the latest CircleCI pipeline for this commit
|
||||
COMMIT_SHA="${{ github.sha }}"
|
||||
echo "Fetching CircleCI results for commit: $COMMIT_SHA"
|
||||
|
||||
# Get pipeline info
|
||||
PIPELINE_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
|
||||
"https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline?branch=main" | \
|
||||
jq -r ".items[] | select(.vcs.revision == \"$COMMIT_SHA\") | .id" | head -1)
|
||||
|
||||
if [ -z "$PIPELINE_INFO" ]; then
|
||||
echo "No CircleCI pipeline found for commit $COMMIT_SHA"
|
||||
echo "Creating placeholder test results..."
|
||||
echo '<?xml version="1.0" encoding="utf-8"?>' > test-results/junit.xml
|
||||
echo '<testsuite name="llm_translation" tests="0" failures="0" errors="0" skipped="0" time="0">' >> test-results/junit.xml
|
||||
echo '<system-out>No CircleCI results found for this commit</system-out>' >> test-results/junit.xml
|
||||
echo '</testsuite>' >> test-results/junit.xml
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Found pipeline: $PIPELINE_INFO"
|
||||
|
||||
# Get workflow info
|
||||
WORKFLOW_ID=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
|
||||
"https://circleci.com/api/v2/pipeline/$PIPELINE_INFO/workflow" | \
|
||||
jq -r '.items[] | select(.name | contains("test")) | .id' | head -1)
|
||||
|
||||
if [ -z "$WORKFLOW_ID" ]; then
|
||||
echo "No test workflow found in pipeline"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found workflow: $WORKFLOW_ID"
|
||||
|
||||
# Get job info for llm_translation tests
|
||||
JOB_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
|
||||
"https://circleci.com/api/v2/workflow/$WORKFLOW_ID/job" | \
|
||||
jq -r '.items[] | select(.name | contains("llm_translation")) | select(.status == "success" or .status == "failed") | .job_number' | head -1)
|
||||
|
||||
if [ -z "$JOB_INFO" ]; then
|
||||
echo "No completed llm_translation job found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found job: $JOB_INFO"
|
||||
|
||||
# Download artifacts
|
||||
curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
|
||||
"https://circleci.com/api/v2/project/github/BerriAI/litellm/$JOB_INFO/artifacts" | \
|
||||
jq -r '.items[] | select(.path | contains("junit") or contains("coverage") or contains("report")) | .url' | \
|
||||
while read -r artifact_url; do
|
||||
filename=$(basename "$artifact_url" | sed 's/[?&].*//')
|
||||
echo "Downloading artifact: $filename"
|
||||
curl -s -H "Circle-Token: $CIRCLE_TOKEN" -o "test-results/$filename" "$artifact_url"
|
||||
done
|
||||
|
||||
# If no artifacts found, create placeholder
|
||||
if [ ! -f "test-results/junit.xml" ]; then
|
||||
echo "No test artifacts found, creating placeholder..."
|
||||
echo '<?xml version="1.0" encoding="utf-8"?>' > test-results/junit.xml
|
||||
echo '<testsuite name="llm_translation" tests="0" failures="0" errors="0" skipped="0" time="0">' >> test-results/junit.xml
|
||||
echo '<system-out>Test artifacts not available from CircleCI</system-out>' >> test-results/junit.xml
|
||||
echo '</testsuite>' >> test-results/junit.xml
|
||||
fi
|
||||
continue-on-error: true
|
||||
|
||||
- name: Generate test summary
|
||||
run: |
|
||||
echo "# LLM Translation Testing Results" > test-results/summary.md
|
||||
echo "" >> test-results/summary.md
|
||||
echo "**Release Candidate:** ${{ github.event.inputs.release_candidate_tag || github.ref_name }}" >> test-results/summary.md
|
||||
echo "**Fetched Date:** $(date)" >> test-results/summary.md
|
||||
echo "**Commit:** ${{ github.sha }}" >> test-results/summary.md
|
||||
echo "**Source:** CircleCI Pipeline" >> test-results/summary.md
|
||||
echo "" >> test-results/summary.md
|
||||
|
||||
# Parse junit.xml for test statistics if it exists
|
||||
if [ -f "test-results/junit.xml" ]; then
|
||||
python -c "
|
||||
import xml.etree.ElementTree as ET
|
||||
try:
|
||||
tree = ET.parse('test-results/junit.xml')
|
||||
root = tree.getroot()
|
||||
tests = root.get('tests', '0')
|
||||
failures = root.get('failures', '0')
|
||||
errors = root.get('errors', '0')
|
||||
skipped = root.get('skipped', '0')
|
||||
time = root.get('time', '0')
|
||||
|
||||
print(f'**Total Tests:** {tests}')
|
||||
print(f'**Passed:** {int(tests) - int(failures) - int(errors) - int(skipped)}')
|
||||
print(f'**Failed:** {failures}')
|
||||
print(f'**Errors:** {errors}')
|
||||
print(f'**Skipped:** {skipped}')
|
||||
print(f'**Duration:** {time} seconds')
|
||||
except Exception as e:
|
||||
print(f'Could not parse test results: {e}')
|
||||
" >> test-results/summary.md
|
||||
fi
|
||||
|
||||
echo "" >> test-results/summary.md
|
||||
echo "## Test Files Covered" >> test-results/summary.md
|
||||
ls tests/llm_translation/*.py | sed 's/^/- /' >> test-results/summary.md
|
||||
|
||||
- name: Upload test results
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: llm-translation-test-results-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
|
||||
path: |
|
||||
test-results/
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
.coverage
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload JUnit test results
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: junit-xml-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
|
||||
path: test-results/junit.xml
|
||||
retention-days: 30
|
||||
Reference in New Issue
Block a user