litellm/.github/workflows/llm-translation-testing.yml

name: LLM Translation Tests

on:
  workflow_dispatch:
    inputs:
      release_candidate_tag:
        description: "Release candidate tag/version"
        required: true
        type: string
  push:
    tags:
      - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1

permissions:
  contents: read

jobs:
  run-llm-translation-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 90

    steps:
      - name: Checkout code
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false
          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.11"

      - name: Install Poetry
        run: |
          pip install 'poetry==2.3.2'
          poetry config virtualenvs.create true
          poetry config virtualenvs.in-project true

      - name: Restore Poetry dependencies cache
        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
        with:
          path: |
            ~/.cache/pypoetry
            .venv
          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
          restore-keys: |
            ${{ runner.os }}-poetry-

      - name: Install dependencies
        run: |
          poetry install --with dev
          poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0'

      - name: Create test results directory
        run: mkdir -p test-results

      - name: Run LLM Translation Tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
          RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
          COMMIT_SHA: ${{ github.sha }}
        run: |
          python .github/workflows/run_llm_translation_tests.py \
            --tag "$RC_TAG" \
            --commit "$COMMIT_SHA" \
            || true  # Continue even if tests fail

      - name: Display test summary
        if: always()
        run: |
          if [ -f "test-results/llm_translation_report.md" ]; then
            echo "Test report generated successfully!"
            echo "Artifact will contain:"
            echo "- test-results/junit.xml (JUnit XML results)"
            echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
          else
            echo "Warning: Test report was not generated"
          fi

      - name: Upload test artifacts
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        if: always()
        with:
          name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
          path: test-results/
          retention-days: 30