litellm/.github/workflows/llm-translation-testing.yml

name: LLM Translation Tests

on:
  workflow_dispatch:
    inputs:
      release_candidate_tag:
        description: "Release candidate tag/version"
        required: true
        type: string
  push:
    tags:
      - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1

permissions:
  contents: read

jobs:
  run-llm-translation-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 90

    steps:
      - name: Checkout code
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false
          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.11"

      - name: Set up uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          version: "0.10.9"
          enable-cache: false

      - name: Restore uv dependencies cache
        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: |
            ~/.cache/uv
            .venv
          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-uv-

      - name: Install dependencies
        run: |
          uv sync --frozen

      - name: Create test results directory
        run: mkdir -p test-results

      - name: Run LLM Translation Tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
          RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
          COMMIT_SHA: ${{ github.sha }}
        run: |
          python .github/workflows/run_llm_translation_tests.py \
            --tag "$RC_TAG" \
            --commit "$COMMIT_SHA" \
            || true  # Continue even if tests fail

      - name: Display test summary
        if: always()
        run: |
          if [ -f "test-results/llm_translation_report.md" ]; then
            echo "Test report generated successfully!"
            echo "Artifact will contain:"
            echo "- test-results/junit.xml (JUnit XML results)"
            echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
          else
            echo "Warning: Test report was not generated"
          fi

      - name: Upload test artifacts
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        if: always()
        with:
          name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
          path: test-results/
          retention-days: 30