diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml new file mode 100644 index 0000000000..83566d6eb6 --- /dev/null +++ b/.github/workflows/llm-translation-testing.yml @@ -0,0 +1,157 @@ +name: LLM Translation Test Results for Release Candidates + +on: + workflow_dispatch: + inputs: + release_candidate_tag: + description: 'Release candidate tag/version' + required: true + type: string + push: + tags: + - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1 + +jobs: + fetch-llm-translation-results: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} + + - name: Install CircleCI CLI + run: | + curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash + + - name: Create test results directory + run: mkdir -p test-results + + - name: Fetch LLM Translation Test Results from CircleCI + env: + CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }} + run: | + # Get the latest CircleCI pipeline for this commit + COMMIT_SHA="${{ github.sha }}" + echo "Fetching CircleCI results for commit: $COMMIT_SHA" + + # Get pipeline info + PIPELINE_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline?branch=main" | \ + jq -r ".items[] | select(.vcs.revision == \"$COMMIT_SHA\") | .id" | head -1) + + if [ -z "$PIPELINE_INFO" ]; then + echo "No CircleCI pipeline found for commit $COMMIT_SHA" + echo "Creating placeholder test results..." + echo '' > test-results/junit.xml + echo '' >> test-results/junit.xml + echo 'No CircleCI results found for this commit' >> test-results/junit.xml + echo '' >> test-results/junit.xml + exit 0 + fi + + echo "Found pipeline: $PIPELINE_INFO" + + # Get workflow info + WORKFLOW_ID=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/pipeline/$PIPELINE_INFO/workflow" | \ + jq -r '.items[] | select(.name | contains("test")) | .id' | head -1) + + if [ -z "$WORKFLOW_ID" ]; then + echo "No test workflow found in pipeline" + exit 1 + fi + + echo "Found workflow: $WORKFLOW_ID" + + # Get job info for llm_translation tests + JOB_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/workflow/$WORKFLOW_ID/job" | \ + jq -r '.items[] | select(.name | contains("llm_translation")) | select(.status == "success" or .status == "failed") | .job_number' | head -1) + + if [ -z "$JOB_INFO" ]; then + echo "No completed llm_translation job found" + exit 1 + fi + + echo "Found job: $JOB_INFO" + + # Download artifacts + curl -s -H "Circle-Token: $CIRCLE_TOKEN" \ + "https://circleci.com/api/v2/project/github/BerriAI/litellm/$JOB_INFO/artifacts" | \ + jq -r '.items[] | select(.path | contains("junit") or contains("coverage") or contains("report")) | .url' | \ + while read -r artifact_url; do + filename=$(basename "$artifact_url" | sed 's/[?&].*//') + echo "Downloading artifact: $filename" + curl -s -H "Circle-Token: $CIRCLE_TOKEN" -o "test-results/$filename" "$artifact_url" + done + + # If no artifacts found, create placeholder + if [ ! -f "test-results/junit.xml" ]; then + echo "No test artifacts found, creating placeholder..." + echo '' > test-results/junit.xml + echo '' >> test-results/junit.xml + echo 'Test artifacts not available from CircleCI' >> test-results/junit.xml + echo '' >> test-results/junit.xml + fi + continue-on-error: true + + - name: Generate test summary + run: | + echo "# LLM Translation Testing Results" > test-results/summary.md + echo "" >> test-results/summary.md + echo "**Release Candidate:** ${{ github.event.inputs.release_candidate_tag || github.ref_name }}" >> test-results/summary.md + echo "**Fetched Date:** $(date)" >> test-results/summary.md + echo "**Commit:** ${{ github.sha }}" >> test-results/summary.md + echo "**Source:** CircleCI Pipeline" >> test-results/summary.md + echo "" >> test-results/summary.md + + # Parse junit.xml for test statistics if it exists + if [ -f "test-results/junit.xml" ]; then + python -c " + import xml.etree.ElementTree as ET + try: + tree = ET.parse('test-results/junit.xml') + root = tree.getroot() + tests = root.get('tests', '0') + failures = root.get('failures', '0') + errors = root.get('errors', '0') + skipped = root.get('skipped', '0') + time = root.get('time', '0') + + print(f'**Total Tests:** {tests}') + print(f'**Passed:** {int(tests) - int(failures) - int(errors) - int(skipped)}') + print(f'**Failed:** {failures}') + print(f'**Errors:** {errors}') + print(f'**Skipped:** {skipped}') + print(f'**Duration:** {time} seconds') + except Exception as e: + print(f'Could not parse test results: {e}') + " >> test-results/summary.md + fi + + echo "" >> test-results/summary.md + echo "## Test Files Covered" >> test-results/summary.md + ls tests/llm_translation/*.py | sed 's/^/- /' >> test-results/summary.md + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: llm-translation-test-results-${{ github.event.inputs.release_candidate_tag || github.ref_name }} + path: | + test-results/ + coverage.xml + htmlcov/ + .coverage + retention-days: 30 + + - name: Upload JUnit test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: junit-xml-${{ github.event.inputs.release_candidate_tag || github.ref_name }} + path: test-results/junit.xml + retention-days: 30 \ No newline at end of file